1.2.1: Fix incorrect diacritics normalization

.gitignore

@@ -35,3 +35,6 @@ node_modules
 # V8 generated
 hydrogen-*
 *.asm
+
+# Cache
+.cache_*

.npmignore

@@ -5,6 +5,7 @@ benchmark/
 .gitignore/
 .DS_Store
 .travis.yml
+.cache_*
 
 *.log
 npm-debug.log*

dev/parseDictionary.js

@@ -25,6 +25,26 @@ const chars = {
 };
 const dictTypes = Object.keys(ranges);
 
+function cachedRequest(url) {
+  const fileName = path.basename(url);
+  const cachePath = `.cache_${fileName}`;
+  if (argv.cache !== 'off') {
+    try {
+      const stat = fs.statSync(cachePath);
+      if (Date.now() - stat.mtime.getTime() <= 1 * 24 * 60 * 60 * 1000 /* 1 day */) {
+        console.log('Using cached content for %s', url);
+        return Promise.resolve(fs.readFileSync(cachePath, 'utf-8'));
+      }
+    } catch (ignore) {
+    }
+  }
+  const req = request(url);
+  return req.then(str => {
+    fs.writeFileSync(cachePath, str);
+    return str;
+  });
+}
+
 function parseSource(sourceStr) {
   const ret = [];
   sourceStr.split('\n').forEach(line => {
@@ -53,10 +73,12 @@ function addDict(codePoint, pinyinList) {
       } else if (c.match(/[a-z]/)) {
         return c;
       } else {
+        console.warn('[Code Point = %d] Invalid character `%s` (\\u%s) after applying mapping for `%s`', codePoint, c, ('0000' + c.charCodeAt(0).toString(16)).slice(-4), pinyinRaw);
         valid = false;
         return '';
       }
     }).join('');
+
     if (!valid) {
       return;
     }
@@ -80,21 +102,16 @@ function addDict(codePoint, pinyinList) {
 Promise.resolve()
   .then(() => {
     console.log('Downloading common characters from %s', CHAR_RANGE_URL);
-    return request(CHAR_RANGE_URL);
+    return cachedRequest(CHAR_RANGE_URL);
   })
   .then(rangeStr => {
     const source = parseSource(rangeStr);
     ranges.common = source.map(d => d.codePoint);
     ranges.common.sort();
   })
   .then(() => {
-    if (argv.dict) {
-      console.log('Using local dictionary %s', argv.dict);
-      return fs.readFileSync(argv.dict, 'utf8');
-    } else {
-      console.log('Downloading latest dictionary from %s', DICTIONARY_URL);
-      return request(DICTIONARY_URL);
-    }
+    console.log('Downloading latest dictionary from %s', DICTIONARY_URL);
+    return cachedRequest(DICTIONARY_URL);
   })
   .then(dictStr => {
     console.log('Parsing dictionary...');

dev/symbolMap.js

@@ -7,6 +7,9 @@ module.exports = {
   'é': 'e',
   'ě': 'e',
   'è': 'e',
+  'ế': 'e',
+  'ê': 'e',
+  'ề': 'e',
   'ō': 'o',
   'ó': 'o',
   'ǒ': 'o',
@@ -24,4 +27,11 @@ module.exports = {
   'ǘ': 'v',
   'ǚ': 'v',
   'ǜ': 'v',
+  'ň': 'n',
+  'ń': 'n',
+  'ǹ': 'n',
+  'ḿ': 'm',
+  '\u0300': '',
+  '\u0304': '',
+  '\u030c': '',
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinyinlite",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "description": "Lightweight and Lightning-Fast Pinyin library",
   "main": "index_full.js",
   "browser": "dist/pinyinlite_common.min.js",