From 023ce098ba70062e2e172ede828c5fd0fe22b9be Mon Sep 17 00:00:00 2001 From: Mike D Pilsbury Date: Mon, 30 Apr 2012 15:05:30 +0100 Subject: [PATCH] Add encodings and tests for Turkish. --- .gitignore | 1 + encodings/turkish.js | 34 ++++++++++++++++ test/turkish-test.js | 92 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 encodings/turkish.js create mode 100644 test/turkish-test.js diff --git a/.gitignore b/.gitignore index 042ccf2..fe46877 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules *~ +*sublime-* diff --git a/encodings/turkish.js b/encodings/turkish.js new file mode 100644 index 0000000..6c8ca95 --- /dev/null +++ b/encodings/turkish.js @@ -0,0 +1,34 @@ +// Turkish encodings + +module.exports = { + "windows1254": { + type: "singlebyte", + chars: "\u20AC\uFFFD\u201A\u0192\u201E\u2026\u2020\u2021\u02C6\u2030\u0160\u2039\u0152\uFFFD\uFFFD\uFFFD" + + "\uFFFD\u2018\u2019\u201C\u201D\u2022\u2013\u2014\u02DC\u2122\u0161\u203A\u0153\uFFFD\uFFFD\u0178" + + "\u00A0\u00A1\u00A2\u00A3\u00A4\u00A5\u00A6\u00A7\u00A8\u00A9\u00AA\u00AB\u00AC\u00AD\u00AE\u00AF" + + "\u00B0\u00B1\u00B2\u00B3\u00B4\u00B5\u00B6\u00B7\u00B8\u00B9\u00BA\u00BB\u00BC\u00BD\u00BE\u00BF" + + "\u00C0\u00C1\u00C2\u00C3\u00C4\u00C5\u00C6\u00C7\u00C8\u00C9\u00CA\u00CB\u00CC\u00CD\u00CE\u00CF" + + "\u011E\u00D1\u00D2\u00D3\u00D4\u00D5\u00D6\u00D7\u00D8\u00D9\u00DA\u00DB\u00DC\u0130\u015E\u00DF" + + "\u00E0\u00E1\u00E2\u00E3\u00E4\u00E5\u00E6\u00E7\u00E8\u00E9\u00EA\u00EB\u00EC\u00ED\u00EE\u00EF" + + "\u011F\u00F1\u00F2\u00F3\u00F4\u00F5\u00F6\u00F7\u00F8\u00F9\u00FA\u00FB\u00FC\u0131\u015F\u00FF" + }, + "win1254": "windows1254", + "cp1254": "windows1254", + 1254: "windows1254", + + "iso88599": { + type: "singlebyte", + chars: "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD" + + "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD" + + "\u00A0\u00A1\u00A2\u00A3\u00A4\u00A5\u00A6\u00A7\u00A8\u00A9\u00AA\u00AB\u00AC\u00AD\u00AE\u00AF" + + "\u00B0\u00B1\u00B2\u00B3\u00B4\u00B5\u00B6\u00B7\u00B8\u00B9\u00BA\u00BB\u00BC\u00BD\u00BE\u00BF" + + "\u00C0\u00C1\u00C2\u00C3\u00C4\u00C5\u00C6\u00C7\u00C8\u00C9\u00CA\u00CB\u00CC\u00CD\u00CE\u00CF" + + "\u011E\u00D1\u00D2\u00D3\u00D4\u00D5\u00D6\u00D7\u00D8\u00D9\u00DA\u00DB\u00DC\u0130\u015E\u00DF" + + "\u00E0\u00E1\u00E2\u00E3\u00E4\u00E5\u00E6\u00E7\u00E8\u00E9\u00EA\u00EB\u00EC\u00ED\u00EE\u00EF" + + "\u011F\u00F1\u00F2\u00F3\u00F4\u00F5\u00F6\u00F7\u00F8\u00F9\u00FA\u00FB\u00FC\u0131\u015F\u00FF" + }, + "turkish": "iso88599", + "turkish8": "iso88599", + "cp28599": "iso88599", + 28599: "iso88599" +} diff --git a/test/turkish-test.js b/test/turkish-test.js new file mode 100644 index 0000000..d34dcbd --- /dev/null +++ b/test/turkish-test.js @@ -0,0 +1,92 @@ +var vows = require('vows'), + assert = require('assert'), + iconv = require(__dirname+'/../'); + +var ascii = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f'+ + ' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f'; + +var encodings = [{ + name: "windows1254", + variations: ['windows-1254', 'win-1254', 'win1254', 'cp1254', 'cp-1254', 1254], + strings: { + empty: "", + ascii: ascii, + greek: "€‚ƒ„…†‡ˆ‰Š‹Œ‘’“”•–—˜™š›œŸ¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏĞÑÒÓÔÕÖ×ØÙÚÛÜİŞßàáâãäåæçèéêëìíîïğñòóôõö÷øùúûüışÿ", + untranslatable: "\x81\x8d\x8e\x8f\x90\x9d\x9e" + }, + encodedStrings: { + empty: new Buffer(''), + ascii: new Buffer(ascii, 'binary'), + greek: new Buffer( + '\x80\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c' + + '\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9f' + + '\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xae\xaf' + + '\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf' + + '\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf' + + '\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf' + + '\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef' + + '\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff', + 'binary'), + } +}, { + name: "iso88599", + variations: ['iso-8859-9', 'turkish', 'turkish8', 'cp28599', 'cp-28599', 28599], + strings: { + empty: "", + ascii: ascii, + greek: "¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏĞÑÒÓÔÕÖ×ØÙÚÛÜİŞßàáâãäåæçèéêëìíîïğñòóôõö÷øùúûüışÿ", + untranslatable: + '\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f' + + '\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f' + }, + encodedStrings: { + empty: new Buffer(''), + ascii: new Buffer(ascii, 'binary'), + greek: new Buffer( + '\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xae\xaf' + + '\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf' + + '\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf' + + '\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf' + + '\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef' + + '\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff', + 'binary') + } +}]; + +var testsBatch = {}; +encodings.forEach(function(encoding) { + var enc = encoding.variations[0]; + var key = "greek"; + var tests = { + "Convert to empty buffer": function() { + assert.strictEqual(iconv.toEncoding("", enc).toString('binary'), new Buffer('').toString('binary')); + }, + "Convert from empty buffer": function() { + assert.strictEqual(iconv.fromEncoding(new Buffer(''), enc), ""); + }, + "Convert from buffer": function() { + for (var key in encoding.encodedStrings) + assert.strictEqual(iconv.fromEncoding(encoding.encodedStrings[key], enc), + encoding.strings[key]); + }, + "Convert to buffer": function() { + for (var key in encoding.encodedStrings) + assert.strictEqual(iconv.toEncoding(encoding.strings[key], enc).toString('binary'), + encoding.encodedStrings[key].toString('binary')); + }, + "Try different variations of encoding": function() { + encoding.variations.forEach(function(enc) { + assert.strictEqual(iconv.fromEncoding(encoding.encodedStrings[key], enc), encoding.strings[key]); + assert.strictEqual(iconv.toEncoding(encoding.strings[key], enc).toString('binary'), encoding.encodedStrings[key].toString('binary')); + }); + }, + "Untranslatable chars are converted to defaultCharSingleByte": function() { + var expected = encoding.strings.untranslatable.split('').map(function(c) {return iconv.defaultCharSingleByte; }).join(''); + assert.strictEqual(iconv.toEncoding(encoding.strings.untranslatable, enc).toString('binary'), expected); // Only '?' characters. + } + }; + + testsBatch[encoding.name+":"] = tests; +}); + +vows.describe("Test Turkish encodings").addBatch(testsBatch).export(module);