Support for NEEDAFFIX and continuation classes

cfinke · May 3, 2011 · e1ee921 · e1ee921
1 parent 6799885
commit e1ee921
Show file tree

Hide file tree

Showing 4 changed files with 81 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -34,6 +34,7 @@ Typo.js has full support for the following Hunspell affix flags:
 * ONLYINCOMPOUND
 * KEEPCASE
 * NOSUGGEST
+* NEEDAFFIX
 
 _Note: The manifest.json file in the root directory of the project is there to simplify testing, as it allows you to load all of the files in the Typo project as a Chrome extension. It doesn't have any purpose if you're using Typo.js in your own project._
 

diff --git a/tests/french.html b/tests/french.html
@@ -1,5 +1,6 @@
 <html>
 	<head>
+		<meta charset="UTF-8">
 		<title>Typo.js French Test Suite</title>
 		<link rel="stylesheet" href="http://code.jquery.com/qunit/git/qunit.css" type="text/css" media="screen" />
 		<script type="text/javascript" src="http://code.jquery.com/jquery-latest.js"></script>
@@ -35,9 +36,24 @@
 
 				test("Contractions are recognized", function () {
 					equal(dict.check("j'espère"), true);
-					equal(dict.check("j'espére"), true);
-					equal(dict.check("C'est"), true);
-					equal(dict.check("c'est"), true);
+					equal(dict.check("j'espére"), false);
+				});
+
+				test("Continuation classes", function () {
+					equal(dict.check("l'impedimentum"), true);
+					equal(dict.check("d'impedimentum"), true);
+					equal(dict.check("d'impedimenta"), true);
+					equal(dict.check("espérés"), true);
+					equal(dict.check("espérée"), true);
+					equal(dict.check("espérées"), true);
+					equal(dict.check("qu'espérés"), true);
+					equal(dict.check("qu'espérée"), true);
+					equal(dict.check("qu'espérées"), true);
+				});
+
+				test("NEEDAFFIX is respected", function () {
+					equal(dict.check("esperluette"), false);
+					equal(dict.check("espressivo"), true);
 				});
 			}
 

diff --git a/tests/german.html b/tests/german.html
@@ -1,5 +1,6 @@
 <html>
 	<head>
+		<meta charset="ISO8859-1">
 		<title>Typo.js German Test Suite</title>
 		<link rel="stylesheet" href="http://code.jquery.com/qunit/git/qunit.css" type="text/css" media="screen" />
 		<script type="text/javascript" src="http://code.jquery.com/jquery-latest.js"></script>

diff --git a/typo/typo.js b/typo/typo.js
@@ -164,12 +164,18 @@ Typo.prototype = {
 
 					var lineParts = line.split(/\s+/);
 					var charactersToRemove = lineParts[2];
-					var charactersToAdd = lineParts[3];
+
+					var additionParts = lineParts[3].split("/");
+					var charactersToAdd = additionParts[0];
+					var continuationClasses = this.parseRuleCodes(additionParts[1]);
+
 					var regexToMatch = lineParts[4];
 
 					var entry = {};
 					entry.add = charactersToAdd;
 
+					if (continuationClasses.length > 0) entry.continuationClasses = continuationClasses;
+
 					if (regexToMatch !== ".") {
 						if (ruleType === "SFX") {
 							entry.match = new RegExp(regexToMatch + "$");
@@ -219,6 +225,7 @@ Typo.prototype = {
 				// COMPOUNDMIN
 				// FLAG
 				// KEEPCASE
+				// NEEDAFFIX
 
 				this.flags[ruleType] = definitionParts[1];
 			}
@@ -259,6 +266,8 @@ Typo.prototype = {
 	 */
 
 	_parseDIC : function (data) {
+		data = this._removeDicComments(data);
+
 		var lines = data.split("\n");
 		var dictionaryTable = {};
 
@@ -269,13 +278,15 @@ Typo.prototype = {
 			var parts = line.split("/", 2);
 
 			var word = parts[0];
-			
+
 			// Now for each affix rule, generate that form of the word.
 			if (parts.length > 1) {
 				var ruleCodesArray = this.parseRuleCodes(parts[1]);
 
 				// Save the ruleCodes for compound word situations.
-				dictionaryTable[word] = ruleCodesArray;
+				if (!("NEEDAFFIX" in this.flags) || ruleCodesArray.indexOf(this.flags.NEEDAFFIX) == -1) {
+					dictionaryTable[word] = ruleCodesArray;
+				}
 
 				for (var j = 0, _jlen = ruleCodesArray.length; j < _jlen; j++) {
 					var code = ruleCodesArray[j];
@@ -324,6 +335,35 @@ Typo.prototype = {
 		return dictionaryTable;
 	},
 
+
+	/**
+	 * Removes comment lines and then cleans up blank lines and trailing whitespace.
+	 *
+	 * @param {String} data The data from a .dic file.
+	 * @return {String} The cleaned-up data.
+	 */
+
+	_removeDicComments : function (data) {
+		// I can't find any official documentation on it, but at least the de_DE
+		// dictionary uses tab-indented lines as comments.
+
+		// Remove comments
+		data = data.replace(/^\t.*$/mg, "");
+
+		return data;
+
+		// Trim each line
+		data = data.replace(/^\s\s*/m, '').replace(/\s\s*$/m, '');
+
+		// Remove blank lines.
+		data = data.replace(/\n{2,}/g, "\n");
+
+		// Trim the entire string
+		data = data.replace(/^\s\s*/, '').replace(/\s\s*$/, '');
+
+		return data;
+	},
+
 	parseRuleCodes : function (textCodes) {
 		if (!textCodes) {
 			return [];
@@ -375,6 +415,23 @@ Typo.prototype = {
 				}
 
 				newWords.push(newWord);
+
+				if ("continuationClasses" in entry) {
+					for (var j = 0, _jlen = entry.continuationClasses.length; j < _jlen; j++) {
+						var continuationRule = this.rules[entry.continuationClasses[j]];
+
+						if (continuationRule) {
+							newWords = newWords.concat(this._applyRule(newWord, continuationRule));
+						}
+						/*
+						else {
+							// This shouldn't happen, but it does, at least in the de_DE dictionary.
+							// I think the author mistakenly supplied lower-case rule codes instead 
+							// of upper-case.
+						}
+						*/
+					}
+				}
 			}
 		}