Skip to content
This repository
Browse code

Improved API and added examples

  • Loading branch information...
commit 3d5d2c79cdb6275afd059fe140572223b841bd48 1 parent 5ace01f
Edison E. Abreu authored March 15, 2012
49  README.md
Source Rendered
... ...
@@ -0,0 +1,49 @@
  1
+# codes
  2
+
  3
+Node.js iconv bindings - Streamable, fast and lightweight
  4
+
  5
+## Installing with [npm](http://npmjs.org/)
  6
+
  7
+	npm install codes
  8
+
  9
+## Usage
  10
+
  11
+Encode from one character encoding to another:
  12
+
  13
+	var assert = require('assert');
  14
+	var codes = require('../index');
  15
+
  16
+	// convert from UTF-8 to ISO-8859-1
  17
+	var str = 'É isso ai. Testando acentuação: áéíóú...';
  18
+	var buffer = codes.encode(str, 'ISO-8859-1//TRANSLIT');
  19
+	var str2 = codes.decode(buffer, 'ISO-8859-1');
  20
+	assert.equal(str, str2);
  21
+
  22
+	// stream from LATIN1 to UTF-8 
  23
+	var cs = codes.createStream('UTF-8', 'LATIN1');
  24
+	var str4 = '';
  25
+	cs.on('data', function (data) {
  26
+		var str3 = data.toString();
  27
+		console.log(str3);
  28
+		str4 += str3;
  29
+	});
  30
+	cs.on('end', function () {
  31
+		assert.equal(str4, str);
  32
+		console.log(str4);
  33
+	});
  34
+	cs.on('error', function (error) {
  35
+		console.log(error);
  36
+	});
  37
+	cs.write(buffer.slice(0,1));
  38
+	cs.write(buffer.slice(1,8));
  39
+	cs.write(buffer.slice(8,16));
  40
+	cs.write(buffer.slice(16,24));
  41
+	cs.write(buffer.slice(24, 28));
  42
+	cs.write(buffer.slice(28, 32));
  43
+	cs.write(buffer.slice(32));
  44
+	cs.end();
  45
+
  46
+	// convert from UTF-8 to ASCII
  47
+	var str = 'aéióuçÁEÍOÚC';
  48
+	var ascii = codes.convert(str, 'ASCII//TRANSLIT', 'UTF-8');
  49
+	console.log(str + ' => ' + ascii.toString());
2  deps.sh
@@ -19,7 +19,7 @@ fi
19 19
 
20 20
 cd deps/libiconv-1.14/
21 21
 if [ ! -f Makefile ]; then
22  
-	./configure --prefix=$PREFIX --enable-static
  22
+	./configure --prefix=$PREFIX --enable-static --enable-extra-encodings
23 23
 fi;
24 24
 make -s && make -s install
25 25
 cd -
37  examples/ex01.js
... ...
@@ -0,0 +1,37 @@
  1
+var assert = require('assert');
  2
+var codes = require('../index');
  3
+
  4
+// convert from UTF-8 to ISO-8859-1
  5
+var str = 'É isso ai. Testando acentuação: áéíóú...';
  6
+var buffer = codes.encode(str, 'ISO-8859-1//TRANSLIT');
  7
+var str2 = codes.decode(buffer, 'ISO-8859-1');
  8
+assert.equal(str, str2);
  9
+
  10
+// stream from LATIN1 to UTF-8 
  11
+var cs = codes.createStream('UTF-8', 'LATIN1');
  12
+var str4 = '';
  13
+cs.on('data', function (data) {
  14
+	var str3 = data.toString();
  15
+	console.log(str3);
  16
+	str4 += str3;
  17
+});
  18
+cs.on('end', function () {
  19
+	assert.equal(str4, str);
  20
+	console.log(str4);
  21
+});
  22
+cs.on('error', function (error) {
  23
+	console.log(error);
  24
+});
  25
+cs.write(buffer.slice(0,1));
  26
+cs.write(buffer.slice(1,8));
  27
+cs.write(buffer.slice(8,16));
  28
+cs.write(buffer.slice(16,24));
  29
+cs.write(buffer.slice(24, 28));
  30
+cs.write(buffer.slice(28, 32));
  31
+cs.write(buffer.slice(32));
  32
+cs.end();
  33
+
  34
+// convert from UTF-8 to ASCII
  35
+var str = 'aéióuçÁEÍOÚC';
  36
+var ascii = codes.convert(str, 'ASCII//TRANSLIT', 'UTF-8');
  37
+console.log(str + ' => ' + ascii.toString());
64  lib/index.js → index.js
... ...
@@ -1,23 +1,21 @@
1  
-var bind = require('../build/Release/iconv_binding');
  1
+var bind = require('./build/Release/iconv');
2 2
 var stream = require('stream');
3 3
 var util = require('util');
4 4
 
5  
-function create(options) {
6  
-	var from = options.from || 'UTF-8',
7  
-			to = options.to,
8  
-			size = options.size || 8192,
  5
+function create(to, from, options) {
  6
+	var from = from || 'UTF-8',
  7
+			to = to.split('//'),
  8
+			size = options && options.size || 8192,
9 9
 			iconv;
10 10
 	from = bind.canonicalize(from);
11  
-	to = bind.canonicalize(to);
12  
-	if (options.option)
13  
-		to += options.option;
  11
+	to[0] = bind.canonicalize(to[0]);
  12
+	to = to.join('//');
14 13
 	iconv = new bind.Iconv(to, from);
15 14
 	return {from: from, to: to, size: size, iconv: iconv};
16 15
 }
17 16
 
18  
-function IconvStream(options) {
19  
-	var h = create(options);
20  
-	this.options = options;
  17
+function CodesStream(to, from, options) {
  18
+	var h = create(to, from, options);
21 19
 	this._iconv = h.iconv;
22 20
 	this._input = new Buffer(h.size);
23 21
 	this._input.start = 0;
@@ -28,15 +26,15 @@ function IconvStream(options) {
28 26
 	this.readable = true;
29 27
 	this.writable = true;
30 28
 }
31  
-util.inherits(IconvStream, stream.Stream);
  29
+util.inherits(CodesStream, stream.Stream);
32 30
 
33  
-IconvStream.prototype.write = function (data) {
  31
+CodesStream.prototype.write = function (data) {
34 32
 	if (!this.writable) {
35 33
 		this.emit('error', new Error('stream not writable'))	
36 34
 		return false;
37 35
 	}
38 36
   if (!Buffer.isBuffer(data)) {
39  
-    var encoding = 'utf8';
  37
+    var encoding = 'UTF-8';
40 38
     if (typeof(arguments[1]) == 'string') encoding = arguments[1];
41 39
     data = new Buffer('' + data, encoding);
42 40
   }
@@ -54,7 +52,7 @@ IconvStream.prototype.write = function (data) {
54 52
 	return true;
55 53
 }
56 54
 
57  
-IconvStream.prototype.end = function (data) {
  55
+CodesStream.prototype.end = function (data) {
58 56
 	if (data)
59 57
 		this.write.apply(this, arguments);
60 58
 	if (this.writable) {
@@ -65,7 +63,7 @@ IconvStream.prototype.end = function (data) {
65 63
 	this.writable = false;
66 64
 }
67 65
 
68  
-IconvStream.prototype._write = function (chunk) {
  66
+CodesStream.prototype._write = function (chunk) {
69 67
 	var input = this._input;
70 68
 	chunk.stop = false;
71 69
 	while (chunk.length > 0 && !chunk.stop)
@@ -78,14 +76,13 @@ IconvStream.prototype._write = function (chunk) {
78 76
 	}
79 77
 }
80 78
 
81  
-IconvStream.prototype._convert = function (chunk) {
  79
+CodesStream.prototype._convert = function (chunk) {
82 80
 	var iconv = this._iconv;
83 81
 	var output = this._output;
84 82
 	var r = iconv.convert(chunk, output);
85  
-	var chunkEnd = chunk.length - r.leftBytesIn;
86  
-	var outputEnd = output.length - r.leftBytesOut;
87  
-	var chunk = chunk.slice(chunkEnd);
88  
-	var data = output.slice(0, outputEnd);
  83
+	var offsetIn = r.offsetIn;
  84
+	var chunk = chunk.slice(offsetIn);
  85
+	var data = output.slice(0, r.offsetOut);
89 86
 	chunk.stop = true;
90 87
 	if (data.length)
91 88
 		this.emit('data', data);
@@ -98,7 +95,7 @@ IconvStream.prototype._convert = function (chunk) {
98 95
 		case 'EILSEQ':
99 96
 			var error = new Error(r.error);
100 97
 			error.code = r.errno;
101  
-			chunk = chunk.slice(chunkEnd + 1);
  98
+			chunk = chunk.slice(offsetIn + 1);
102 99
 			this.emit('error', error);
103 100
 			return chunk.slice(0, 0);
104 101
 			break;
@@ -112,18 +109,25 @@ IconvStream.prototype._convert = function (chunk) {
112 109
 	return chunk;
113 110
 }
114 111
 
115  
-exports.createStream = function (options) {
116  
-	return new IconvStream(options);
  112
+exports.CodesStream = CodesStream;
  113
+exports.createStream = function (to, from, options) {
  114
+	return new CodesStream(to, from, options);
117 115
 }
118  
-exports.convert = function (input, options) {
  116
+exports.convert = function (input, to, from, options) {
119 117
 	if (typeof input == 'string')
120  
-		input = new Buffer(input, 'utf8');
121  
-	var h = create(options);
122  
-	var output = new Buffer(input.length * 2);
  118
+		input = new Buffer(input, 'UTF-8');
  119
+	var h = create(to, from, options);
  120
+	var output = new Buffer(input.length * 4);
123 121
 	var r = h.iconv.convert(input, output);
124  
-	var outputEnd = output.length - r.leftBytesOut;
125  
-	return output.slice(0, outputEnd);
  122
+	return output.slice(0, r.offsetOut);
  123
+}
  124
+exports.encode = function (input, to, options) {
  125
+	return this.convert(input, to, 'UTF-8', options);
  126
+}
  127
+exports.decode = function (input, from, options) {
  128
+	return this.convert(input, 'UTF-8', from, options).toString();
126 129
 }
127 130
 exports.encodings = bind.encodings;
  131
+exports.canonicalize = bind.canonicalize;
128 132
 exports.TRANSLIT = '//TRANSLIT';
129 133
 exports.IGNORE = '//IGNORE';
8  package.json
@@ -2,12 +2,14 @@
2 2
   "author": "Edison E. Abreu <dinho.abreu@gmail.com>",
3 3
   "name": "codes",
4 4
   "description": "Node.js iconv bindings - Streamable, fast and lightweight",
5  
-  "version": "0.0.2",
  5
+	"keywords": "iconv charset encoding encode decode convert code codes stream fast ligthweight",
  6
+	"homepage": "https://github.com/dinhoabreu/codes",
  7
+  "version": "0.1.0",
6 8
   "repository": {
7 9
     "type": "git",
8 10
     "url": "git@github.com:dinhoabreu/codes.git"
9 11
   },
10  
-  "main": "lib/index.js",
  12
+  "main": "index.js",
11 13
 	"scripts": {
12 14
 		"prepublish": "node-waf distclean && rm -rf libiconv",
13 15
 		"preinstall": "./deps.sh",
@@ -20,6 +22,6 @@
20 22
 	},
21 23
   "optionalDependencies": {},
22 24
   "engines": {
23  
-    "node": "0.6.x"
  25
+    "node": ">=0.6.x"
24 26
   }
25 27
 }
36  src/iconv_binding.cc → src/iconv.cc
@@ -57,15 +57,15 @@ class Iconv {
57 57
 		static void Initialize(Handle<Object> target) {
58 58
 			HandleScope scope;
59 59
 
60  
-  			push_sym = Persistent<String>::New(String::NewSymbol("push"));
61  
-  			touppercase_sym = Persistent<String>::New(String::NewSymbol("toUpperCase"));
62  
-  			convert_sym = Persistent<String>::New(String::NewSymbol("convert"));
63  
-  			iconv_sym = Persistent<String>::New(String::NewSymbol("Iconv"));
64  
-  			canonicalize_sym = Persistent<String>::New(String::NewSymbol("canonicalize"));
65  
-  			encodings_sym = Persistent<String>::New(String::NewSymbol("encodings"));
66  
-
67  
-			leftBytesIn_sym = Persistent<String>::New(String::NewSymbol("leftBytesIn"));
68  
-			leftBytesOut_sym = Persistent<String>::New(String::NewSymbol("leftBytesOut"));
  60
+  		push_sym = Persistent<String>::New(String::NewSymbol("push"));
  61
+  		touppercase_sym = Persistent<String>::New(String::NewSymbol("toUpperCase"));
  62
+  		convert_sym = Persistent<String>::New(String::NewSymbol("convert"));
  63
+  		iconv_sym = Persistent<String>::New(String::NewSymbol("Iconv"));
  64
+  		canonicalize_sym = Persistent<String>::New(String::NewSymbol("canonicalize"));
  65
+  		encodings_sym = Persistent<String>::New(String::NewSymbol("encodings"));
  66
+
  67
+			offsetIn_sym = Persistent<String>::New(String::NewSymbol("offsetIn"));
  68
+			offsetOut_sym = Persistent<String>::New(String::NewSymbol("offsetOut"));
69 69
 			code_sym = Persistent<String>::New(String::NewSymbol("code"));
70 70
 			errno_sym = Persistent<String>::New(String::NewSymbol("errno"));
71 71
 			error_sym = Persistent<String>::New(String::NewSymbol("error"));
@@ -160,13 +160,15 @@ class Iconv {
160 160
 			Local<Object> target_obj = args[1]->ToObject();
161 161
 			char *source_data = Buffer::Data(source_obj);
162 162
 			size_t source_length = Buffer::Length(source_obj);
  163
+			size_t source_left = source_length;
163 164
 			char *target_data = Buffer::Data(target_obj);
164 165
 			size_t target_length = Buffer::Length(target_obj);
165  
-			size_t r = ic->convert(&source_data, &source_length, &target_data, &target_length);
  166
+			size_t target_left = target_length;
  167
+			size_t r = ic->convert(&source_data, &source_left, &target_data, &target_left);
166 168
 
167 169
 			Local<Object> obj = Object::New();
168  
-			obj->Set(leftBytesIn_sym, Integer::NewFromUnsigned(source_length));
169  
-			obj->Set(leftBytesOut_sym, Integer::NewFromUnsigned(target_length));
  170
+			obj->Set(offsetIn_sym, Integer::NewFromUnsigned(source_length - source_left));
  171
+			obj->Set(offsetOut_sym, Integer::NewFromUnsigned(target_length - target_left));
170 172
 			obj->Set(code_sym, Integer::New(r));
171 173
 			if (r == -1) {
172 174
 				obj->Set(error_sym, String::New(strerror(errno)));
@@ -184,8 +186,8 @@ class Iconv {
184 186
 			return scope.Close(obj);
185 187
 		}
186 188
 		static Persistent<FunctionTemplate> constructor_template;
187  
-		static Persistent<String> leftBytesIn_sym;
188  
-		static Persistent<String> leftBytesOut_sym;
  189
+		static Persistent<String> offsetIn_sym;
  190
+		static Persistent<String> offsetOut_sym;
189 191
 		static Persistent<String> code_sym;
190 192
 		static Persistent<String> errno_sym;
191 193
 		static Persistent<String> error_sym;
@@ -194,8 +196,8 @@ class Iconv {
194 196
 };
195 197
 
196 198
 Persistent<FunctionTemplate> Iconv::constructor_template;
197  
-Persistent<String> Iconv::leftBytesIn_sym;
198  
-Persistent<String> Iconv::leftBytesOut_sym;
  199
+Persistent<String> Iconv::offsetIn_sym;
  200
+Persistent<String> Iconv::offsetOut_sym;
199 201
 Persistent<String> Iconv::code_sym;
200 202
 Persistent<String> Iconv::errno_sym;
201 203
 Persistent<String> Iconv::error_sym;
@@ -203,4 +205,4 @@ Persistent<String> Iconv::error_sym;
203 205
 void init(Handle<Object> target) {
204 206
 	Iconv::Initialize(target);
205 207
 }
206  
-NODE_MODULE(iconv_binding, init)
  208
+NODE_MODULE(iconv, init)
48  test/index.js
... ...
@@ -1,7 +1,7 @@
1  
-var vows = require('vows');
2 1
 var assert = require('assert');
  2
+var vows = require('vows');
3 3
 var events = require('events');
4  
-var iconv = require('../lib/index');
  4
+var codes = require('../index');
5 5
 
6 6
 function write(str, stream) {
7 7
 	var data = new Buffer(str);
@@ -10,14 +10,18 @@ function write(str, stream) {
10 10
 }
11 11
 
12 12
 vows.describe('Convert charset by Codes').addBatch({
  13
+	'UTF8 to ASCII': {
  14
+		topic: function () {
  15
+			return codes.createStream('ascii', 'utf-8');
  16
+		},
  17
+		'when got CodesStream instance': function (ex) {
  18
+			assert.instanceOf(ex, codes.CodesStream);
  19
+		}
  20
+	},
13 21
 	'UTF-8 to ASCII': {
14 22
 		'without TRANSLIT': {
15 23
 			topic: function () {
16  
-				var options = {
17  
-					from: 'utf-8',
18  
-					to: 'ascii'
19  
-				};
20  
-				return iconv.createStream(options);
  24
+				return codes.createStream('ascii', 'utf-8');
21 25
 			},
22 26
 			'when write "áéíóúç"': {
23 27
 				topic: function (stream) {
@@ -41,12 +45,7 @@ vows.describe('Convert charset by Codes').addBatch({
41 45
 		},
42 46
 		'with TRANSLIT': {
43 47
 			topic: function () {
44  
-				var options = {
45  
-					from: 'utf-8',
46  
-					to: 'ascii',
47  
-					option: iconv.TRANSLIT
48  
-				};
49  
-				return iconv.createStream(options);
  48
+				return codes.createStream('ascii//TRANSLIT', 'utf-8');
50 49
 			},
51 50
 			'when write "áéíóúç"': {
52 51
 				topic: function (stream) {
@@ -72,11 +71,7 @@ vows.describe('Convert charset by Codes').addBatch({
72 71
 	'UTF-8 to Latin1': {
73 72
 		'without TRANSLIT': {
74 73
 			topic: function () {
75  
-				var options = {
76  
-					from: 'utf-8',
77  
-					to: 'latin1'
78  
-				};
79  
-				return iconv.createStream(options);
  74
+				return codes.createStream('latin1', 'utf-8');
80 75
 			},
81 76
 			'when write "áéíóú"': {
82 77
 				topic: function (stream) {
@@ -100,12 +95,7 @@ vows.describe('Convert charset by Codes').addBatch({
100 95
 		},
101 96
 		'with TRANSLIT': {
102 97
 			topic: function () {
103  
-				var options = {
104  
-					from: 'utf-8',
105  
-					to: 'latin1',
106  
-					option: iconv.TRANSLIT
107  
-				};
108  
-				return iconv.createStream(options);
  98
+				return codes.createStream('latin1//TRANSLIT', 'utf-8');
109 99
 			},
110 100
 			'when write "áéíóú"': {
111 101
 				topic: function (stream) {
@@ -130,11 +120,7 @@ vows.describe('Convert charset by Codes').addBatch({
130 120
 	},
131 121
 	'UTF-8 to UTF-16': {
132 122
 		topic: function () {
133  
-			var options = {
134  
-				from: 'utf-8',
135  
-				to: 'utf-16'
136  
-			};
137  
-			return iconv.createStream(options);
  123
+			return codes.createStream('utf-16', 'utf-8');
138 124
 		},
139 125
 		'streaming file "input.txt"': {
140 126
 			topic: function (stream) {
@@ -146,9 +132,7 @@ vows.describe('Convert charset by Codes').addBatch({
146 132
 				stream.on('error', function (error) {
147 133
 					promise.emit('error', error);
148 134
 				});
149  
-				process.nextTick(function () {
150  
-					reader.pipe(stream);
151  
-				});
  135
+				reader.pipe(stream);
152 136
 				return promise;
153 137
 			},
154 138
 			"we get hex('feff00e100e900ed00f300fa000a00e7000a')": function (error, data) {
4  wscript
@@ -19,6 +19,6 @@ def configure(conf):
19 19
 def build(bld):
20 20
   iconv = bld.new_task_gen('cxx', 'shlib', 'node_addon')
21 21
   iconv.includes = '/usr/include /usr/local/include src'
22  
-  iconv.target = 'iconv_binding'
23  
-  iconv.source = 'src/iconv_binding.cc'
  22
+  iconv.target = 'iconv'
  23
+  iconv.source = 'src/iconv.cc'
24 24
   iconv.uselib = "iconv"

0 notes on commit 3d5d2c7

Please sign in to comment.
Something went wrong with that request. Please try again.