Merge pull request ashtuchkin#10 from pekim/master

Add support for Turkish encodings
fengmk2 · May 6, 2012 · b7fb337 · b7fb337
2 parents e40d82d + 023ce09
commit b7fb337
Show file tree

Hide file tree

Showing 3 changed files with 127 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,3 @@
 node_modules
 *~
+*sublime-*
diff --git a/encodings/turkish.js b/encodings/turkish.js
@@ -0,0 +1,34 @@
+// Turkish encodings
+
+module.exports = {
+    "windows1254": {
+        type: "singlebyte",
+        chars: "\u20AC\uFFFD\u201A\u0192\u201E\u2026\u2020\u2021\u02C6\u2030\u0160\u2039\u0152\uFFFD\uFFFD\uFFFD" +
+                "\uFFFD\u2018\u2019\u201C\u201D\u2022\u2013\u2014\u02DC\u2122\u0161\u203A\u0153\uFFFD\uFFFD\u0178" +
+                "\u00A0\u00A1\u00A2\u00A3\u00A4\u00A5\u00A6\u00A7\u00A8\u00A9\u00AA\u00AB\u00AC\u00AD\u00AE\u00AF" +
+                "\u00B0\u00B1\u00B2\u00B3\u00B4\u00B5\u00B6\u00B7\u00B8\u00B9\u00BA\u00BB\u00BC\u00BD\u00BE\u00BF" +
+                "\u00C0\u00C1\u00C2\u00C3\u00C4\u00C5\u00C6\u00C7\u00C8\u00C9\u00CA\u00CB\u00CC\u00CD\u00CE\u00CF" +
+                "\u011E\u00D1\u00D2\u00D3\u00D4\u00D5\u00D6\u00D7\u00D8\u00D9\u00DA\u00DB\u00DC\u0130\u015E\u00DF" +
+                "\u00E0\u00E1\u00E2\u00E3\u00E4\u00E5\u00E6\u00E7\u00E8\u00E9\u00EA\u00EB\u00EC\u00ED\u00EE\u00EF" +
+                "\u011F\u00F1\u00F2\u00F3\u00F4\u00F5\u00F6\u00F7\u00F8\u00F9\u00FA\u00FB\u00FC\u0131\u015F\u00FF"
+    },
+    "win1254": "windows1254",
+    "cp1254": "windows1254",
+    1254: "windows1254",
+
+    "iso88599": {
+        type: "singlebyte",
+        chars: "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD" +
+                "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD" +
+                "\u00A0\u00A1\u00A2\u00A3\u00A4\u00A5\u00A6\u00A7\u00A8\u00A9\u00AA\u00AB\u00AC\u00AD\u00AE\u00AF" +
+                "\u00B0\u00B1\u00B2\u00B3\u00B4\u00B5\u00B6\u00B7\u00B8\u00B9\u00BA\u00BB\u00BC\u00BD\u00BE\u00BF" +
+                "\u00C0\u00C1\u00C2\u00C3\u00C4\u00C5\u00C6\u00C7\u00C8\u00C9\u00CA\u00CB\u00CC\u00CD\u00CE\u00CF" +
+                "\u011E\u00D1\u00D2\u00D3\u00D4\u00D5\u00D6\u00D7\u00D8\u00D9\u00DA\u00DB\u00DC\u0130\u015E\u00DF" +
+                "\u00E0\u00E1\u00E2\u00E3\u00E4\u00E5\u00E6\u00E7\u00E8\u00E9\u00EA\u00EB\u00EC\u00ED\u00EE\u00EF" +
+                "\u011F\u00F1\u00F2\u00F3\u00F4\u00F5\u00F6\u00F7\u00F8\u00F9\u00FA\u00FB\u00FC\u0131\u015F\u00FF"
+    },
+    "turkish": "iso88599",
+    "turkish8": "iso88599",
+    "cp28599": "iso88599",
+    28599: "iso88599"
+}
diff --git a/test/turkish-test.js b/test/turkish-test.js
@@ -0,0 +1,92 @@
+var vows = require('vows'),
+    assert = require('assert'),
+    iconv = require(__dirname+'/../');
+
+var ascii = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f'+
+           ' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f';
+
+var encodings = [{
+    name: "windows1254",
+    variations: ['windows-1254', 'win-1254', 'win1254', 'cp1254', 'cp-1254', 1254],
+    strings: {
+        empty: "",
+        ascii: ascii,
+        greek: "€‚ƒ„…†‡ˆ‰Š‹Œ‘’“”•–—˜™š›œŸ¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏĞÑÒÓÔÕÖ×ØÙÚÛÜİŞßàáâãäåæçèéêëìíîïğñòóôõö÷øùúûüışÿ",
+        untranslatable: "\x81\x8d\x8e\x8f\x90\x9d\x9e"
+    },
+    encodedStrings: {
+        empty: new Buffer(''),
+        ascii: new Buffer(ascii, 'binary'),
+        greek: new Buffer(
+            '\x80\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c' +
+            '\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9f' +
+            '\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xae\xaf' +
+            '\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf' +
+            '\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf' +
+            '\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf' +
+            '\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef' +
+            '\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff',
+            'binary'),
+    }
+}, {
+    name: "iso88599",
+    variations: ['iso-8859-9', 'turkish', 'turkish8', 'cp28599', 'cp-28599', 28599],
+    strings: {
+        empty: "",
+        ascii: ascii,
+        greek: "¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏĞÑÒÓÔÕÖ×ØÙÚÛÜİŞßàáâãäåæçèéêëìíîïğñòóôõö÷øùúûüışÿ",
+        untranslatable:
+            '\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f' +
+            '\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f'
+    },
+    encodedStrings: {
+        empty: new Buffer(''),
+        ascii: new Buffer(ascii, 'binary'),
+        greek: new Buffer(
+            '\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xae\xaf' +
+            '\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf' +
+            '\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf' +
+            '\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf' +
+            '\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef' +
+            '\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff',
+             'binary')
+    }
+}];
+
+var testsBatch = {};
+encodings.forEach(function(encoding) {
+    var enc = encoding.variations[0];
+    var key = "greek";
+    var tests = {
+        "Convert to empty buffer": function() {
+            assert.strictEqual(iconv.toEncoding("", enc).toString('binary'), new Buffer('').toString('binary'));
+        },
+        "Convert from empty buffer": function() {
+            assert.strictEqual(iconv.fromEncoding(new Buffer(''), enc), "");
+        },
+        "Convert from buffer": function() {
+            for (var key in encoding.encodedStrings)
+                assert.strictEqual(iconv.fromEncoding(encoding.encodedStrings[key], enc),
+                    encoding.strings[key]);
+        },
+        "Convert to buffer": function() {
+            for (var key in encoding.encodedStrings)
+                assert.strictEqual(iconv.toEncoding(encoding.strings[key], enc).toString('binary'),
+                    encoding.encodedStrings[key].toString('binary'));
+        },
+        "Try different variations of encoding": function() {
+            encoding.variations.forEach(function(enc) {
+                assert.strictEqual(iconv.fromEncoding(encoding.encodedStrings[key], enc), encoding.strings[key]);
+                assert.strictEqual(iconv.toEncoding(encoding.strings[key], enc).toString('binary'), encoding.encodedStrings[key].toString('binary'));
+            });
+        },
+        "Untranslatable chars are converted to defaultCharSingleByte": function() {
+            var expected = encoding.strings.untranslatable.split('').map(function(c) {return iconv.defaultCharSingleByte; }).join('');
+            assert.strictEqual(iconv.toEncoding(encoding.strings.untranslatable, enc).toString('binary'), expected); // Only '?' characters.
+        }
+    };
+
+    testsBatch[encoding.name+":"] = tests;
+});
+
+vows.describe("Test Turkish encodings").addBatch(testsBatch).export(module);