Permalink
Browse files

Add first src codes.

  • Loading branch information...
1 parent e30d64b commit aa8ecb44928a0abcf22b3273817cc79bc1f68899 @mxfli committed Jan 11, 2012
Showing with 508 additions and 2 deletions.
  1. +13 −2 README.md
  2. +13 −0 bin/config.js
  3. +10 −0 index.js
  4. +17 −0 lib/cookie.js
  5. +29 −0 lib/headers.js
  6. +69 −0 lib/logger.js
  7. +84 −0 lib/request.pipe.js
  8. +22 −0 lib/statusCode.js
  9. +19 −0 package.json
  10. +51 −0 streamPlugins/streamIconv.js
  11. +7 −0 streamPlugins/streamJSON.js
  12. +7 −0 streamPlugins/streamProtobuf.js
  13. +21 −0 test/cookies.txt
  14. +54 −0 test/requestTest.js
  15. +92 −0 test/server.js
View
@@ -1,8 +1,19 @@
# About request.pipe
-rquest.pipe is a piped request module for node.js
+rquest.pipe is a piped chain request module for node.js
* You can use it like pipe(stream1).pipe(stream2).pipe(stream-X);
* Chaining stream for request and response
* connect style router pipes for request and response depends on request and response headers;
-It`s coming ... slowlly...^_^
+It`s here ... now!...^_^
+
+##Usage
+1. install module request.pipe: npm install request.pipe (not push tu npm repo);
+2. use bin/config.js enable desable exists plugins;
+2. write your own strem module or use exits stream module. see example:./streamPlugins/streamIconv.js
+3. add stream to routers by order. request.addRouter();
+4. usage example: ./test/requestTest.js
+
+##TODO List
+ * Improve interface and fix bugs.
+ * Refactor and improve documents.
View
@@ -0,0 +1,13 @@
+/**
+ * FileName: config.js
+ * Author: @mxfli
+ * CreateTime: 2012-01-10 22:30
+ * Description:
+ * module config and setup script.Index.js and pakage.json generator.
+ */
+//TODO(Inaction) config package.json and update whith npm
+//Default package.json is no plugin installed.
+// 1. list plugins avalible and installed.
+// 2. install all,remove all, install? remove? connond select;
+// 3. input install modeue.
+// 4. npm update;
View
@@ -0,0 +1,10 @@
+/**
+ * FileName: index.js
+ * Author: @mxfli
+ * CreateTime: 2012-01-03 23:42
+ * Description:
+ * Description of index.js
+ */
+var requestPipe = require('lib/request.pipe.js');
+
+exports['request.pipe'] = requestPipe;
View
@@ -0,0 +1,17 @@
+/**
+ * FileName: cookie.js
+ * Author: @mxfli
+ * CreateTime: 2012-01-03 23:43
+ * Description:
+ * Description of cookie.js
+ */
+var cookie = function () {
+ //Write codes here
+ //load cookies from cookies.txt
+ //update cookies in memory?
+ //keeping cookies for other requests?
+ //write cookies to cookies.txt?
+}();
+
+module.exports = cookie;
+
View
@@ -0,0 +1,29 @@
+/**
+ * FileName: headers.js
+ * Author: @mxfli
+ * CreateTime: 2012-01-03 23:43
+ * Description:
+ * Parse request and response headers;
+ */
+var headers = exports;
+
+headers.addDefaultHeaders = function (options) {
+ options.headers = options.headers || {};
+ options.headers['Accept-Charset'] = 'UTF-8,*;q=0.5';//from chrome
+ options.headers['Accept-Encoding'] = 'gzip'; // accept gzip content-encoding
+ options.headers['UserAgent'] = 'Request.pipe v0.0.1 node.js(' + process.version + ')';
+};
+
+//Call this by func.call(this, name, value)
+headers.setHeader = function (name, value) {
+ this[name] = value;
+};
+
+headers.parseHeaders = function (request, response) {
+ //gzip
+ //mutipart? 断点续传 后期再写
+ //GBK to utf-8
+ //SAVE to File
+ //parse page:SAX
+ //end this request.
+};
View
@@ -0,0 +1,69 @@
+/**
+ * FileName: logger.js
+ * Author: @mxfli
+ * CreateTime: 2012-01-10 19:52
+ * Description:
+ * Simple logger for nodejs.
+ */
+
+var levels = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG"];
+
+if (levels.indexOf(process.env['LOG_LEVEL']) === -1) {
+ console.error('ERROR : ', 'Your value:', process.env['LOG_LEVEL'], " accepted LOG_LEVEL values: ", levels.join(';'));
+ throw new Error('Available process.env["LOG_LEVEL"] value.');
+}
+
+var LOG_LEVEL = process.env['LOG_LEVEL'] || levels[3]; //DEFAULT level is 1:INFO;
+var levelLogger = {
+ "FATAL":console.error.bind(null, 'FATAL :'),
+ "ERROR":console.error.bind(null, 'ERROR :'),
+ "WARN":console.warn.bind(null, 'WARN :'),
+ "INFO":console.log.bind(null, 'INFO :'),
+ "DEBUG":console.log.bind(null, 'DEBUG :')
+};
+
+/**
+ * real log function;
+ * this value for log level: this.level
+ */
+var log = function () {
+ switch (arguments.length) {
+ case 0:
+ levels[this.level] >= 3 && console.log();
+ break;
+ default:
+ var level = LOG_LEVEL;
+ var arg0 = arguments[0];
+ var sliceIndex = 0;
+ if (typeof arg0 === 'string') {
+ arg0 = arg0.trim().toUpperCase();
+ if (levels.indexOf(arg0) !== -1) {
+ sliceIndex = 1;
+ level = arg0;
+ }
+ }
+ if (levels.indexOf(this.level) >= levels.indexOf(level)) {
+ levelLogger[level].apply(null, Array.prototype.slice.call(arguments, sliceIndex));
+ }
+ }
+};
+
+var logger = exports;
+
+/**
+ * process level log
+ * set log level use process.env['LOG_LEVEL'], default is level:INFO
+ */
+logger.log = function () { log.apply({level:LOG_LEVEL}, arguments)};
+
+/**
+ * Custom level for debug is usfule.
+ * @param level Custom level bind to the logger.default is process.env['LOG_LEVEL']
+ */
+logger.getLogger = function (level) {
+ console.assert(typeof level === 'string');
+ level = level.trim().toUpperCase();
+ console.assert(levels.indexOf(level) !== -1);
+ level = level || LOG_LEVEL;
+ return {log:log.bind({level:level})};
+};
View
@@ -0,0 +1,84 @@
+/**
+ * FileName: request.pipe.js
+ * Author: @mxfli
+ * CreateTime: 2012-01-04 12:29
+ * Description:
+ * Description of request.pipe.js
+ * use: muti-thread download/crawler/mini memory usage
+ * A request module that use pipe().pie() style codes, and router
+ * like connect depends on request urls and response headers.
+ */
+
+var http = require('http');
+var globalAgent = http.globalAgent;
+var url = require('url');
+var path = require('path');
+var zlib = require('zlib');
+var util = require('util');
+var logger = require('./logger.js');//.getLogger('debug');
+var headers = require('./headers.js');
+var statusCode = require('./statusCode.js');
+
+var pipedRequest = function () {
+ globalAgent.maxSockets = 1; //set maxSockets=2 in dev mod.
+ //var reqType = ['link', 'js', 'css', 'image', 'ota/stream'];
+ var routers = [];//{type:'in reqType',options:optons}
+
+ var that = {};
+
+ //every request is an cpmpleate session.
+ that.get = function (options, callback) {
+ logger.log('debug', 'request Headers:\n', util.inspect(options.headers));
+ headers.addDefaultHeaders(options);
+ //TODO(Inaction) Add cookies support.
+ logger.log('warn', 'Cookies is not stetted.');
+ if (options.cookie) {
+ headers.setHeader.call(options.headers, 'cookie', options.cookie.getString());
+ }
+
+ var clientRequest = http.get(options);
+ clientRequest.on('error', function (err) {callback(err)});
+ clientRequest.on('response', function (response) {
+ response.pause();
+
+ //read statusCode and headers do something.
+ statusCode.doSomething(options, response, function () {
+ logger.log('info', 'call statusCode module.');
+ //next() function. Add next to router.
+ });
+
+ logger.log('info', 'response Headers:\n', util.inspect(response.headers, true, 3, true));
+
+ console.assert(Array.isArray(routers) && routers.length > 0);
+ var raw = response;
+
+ if (response.headers['content-encoding'] === 'gzip') {
+ raw = response.pipe(zlib.createGunzip());
+ }
+
+ routers.forEach(function (router, index) {
+ logger.log('debug', 'Add router:', index);
+ var _raw = router(options, response);
+ _raw && (raw = raw.pipe(_raw));
+ });
+
+ response.resume();
+ });
+ };
+
+ /**
+ * Add router rules to request.pipe
+ * @param router Router function whith three arguments.
+ * arg1: clientRequest options;
+ * arg2: clientResponse object;
+ * arg3: next router caller;
+ *
+ */
+ that.addRouter = function (router) {
+ routers.push(router);
+ };
+
+ return that;
+}();
+
+module.exports = pipedRequest;
View
@@ -0,0 +1,22 @@
+/**
+ * FileName: statusCode.js
+ * Author: @mxfli
+ * CreateTime: 2012-01-08 11:19
+ * Description:
+ * Parse response statuscode
+ */
+var logger = require('./logger.js');
+var statusCode = exports;
+
+statusCode.doSomething = function (request, response, next) {
+ var code = response.statusCode;
+ logger.log('debug', 'response statusCode:', code);
+
+ if (code === 200) {
+ next();
+ } else if (code > 300 && code < 400) {
+ throw new Error("Redirect is not implement." + code);
+ } else {
+ throw new Error('Wrong statusCode for request:' + code);
+ }
+};
View
@@ -0,0 +1,19 @@
+{ "name":"request.pipe",
+ "description":"pipe().pipe() request and response and router like connect.",
+ "tags":["request", "pipe", "http", "stream"],
+ "version":"0.0.1", "author":"Inaction <Inaction.me@gmail.com> http://inaction.me",
+ "repository":{
+ "type":"git",
+ "url":"https://github.com/mxfli/request.pipe.git"
+ },
+ "bugs":{
+ "url":"https://github.com/mxfli/request.pipe/issues"
+ },
+ "dependencies":{
+ "iconv":">=1.1.x"
+ },
+ "engines":{
+ "node":">= 0.6.x < 0.7.0"
+ },
+ "main":"index"
+}
@@ -0,0 +1,51 @@
+/**
+ * FileName: streamIconv.js
+ * Author: @mxfli
+ * CreateTime: 2011-12-17 20:48
+ * Description:
+ * Stream ICONV convert for node.
+ * node-iconv have Stream module whith a stream branch, but not released.
+ */
+var Stream = require('stream').Stream;
+var util = require('util');
+var Iconv = require('iconv').Iconv;
+var logger = require('../lib/logger.js');//.getLogger('DEBUG');
+
+function StreamIconv(convert) {
+ Stream.call(this);
+ this.convert = function (chunk) {return convert(chunk)};
+ this.readable = true;
+ this.writable = true;
+ this.buffers = [];
+ this.length = 0;
+ this.needChangeCharset = true;
+}
+util.inherits(StreamIconv, Stream);
+
+StreamIconv.prototype.write = function (data) {
+ //GBK first char:0x81~0xFE; second char:0x40~0xFE
+ //TODO(Inaction) less memory useage mode,add real stream convert.
+ this.buffers.push(data);
+ this.length += data.length;
+ logger.log('debug', 'Total data lenth:', this.length);
+};
+
+StreamIconv.prototype.end = function () {
+ //这样一次性转换性能好些,但内存消耗可能会比较多。
+ var buffer = new Buffer(this.length);
+ var index = 0;
+ this.buffers.forEach(function (chunk) {
+ chunk.copy(buffer, index);
+ index += chunk.length
+ });
+
+ //Send data to write stream at once.
+ this.emit('data', this.convert(buffer));
+ this.emit('end');
+};
+
+exports.createIconvStream = function (inputEncoding, outputEncoding) {
+ logger.log('Create Iconv Stream.', inputEncoding, 'to', outputEncoding);
+ var iconv = new Iconv(inputEncoding, outputEncoding + '//IGNORE');
+ return new StreamIconv(iconv.convert.bind(iconv));
+};
@@ -0,0 +1,7 @@
+/**
+ * FileName: streamJSON.js
+ * Author: @mxfli
+ * CreateTime: 2012-01-10 22:24
+ * Description:
+ * Description of streamJSON.js
+ */
@@ -0,0 +1,7 @@
+/**
+ * FileName: streamProtobuf.js
+ * Author: @mxfli
+ * CreateTime: 2012-01-10 22:24
+ * Description:
+ * Description of streamProtobuf.js
+ */
View
@@ -0,0 +1,21 @@
+# this is a test.
+# this is at common too.
+store.ngotcm.com FALSE /mall FALSE 1326330661 S[FIRST_REFER] %7B%22ID%22%3A%22%22%2C%22REFER%22%3A%22http%3A%2F%2Fwww.ngotcm.com%2Fforum%2Fforum.php%22%2C%22DATE%22%3A1325034664000%7D
+store.ngotcm.com FALSE /mall FALSE 1326330661 S[NOW_REFER] %7B%22ID%22%3A%22%22%2C%22REFER%22%3A%22http%3A%2F%2Fwww.ngotcm.com%2Fforum%2Fforum.php%22%2C%22DATE%22%3A1325034664000%7D
+store.ngotcm.com FALSE /mall FALSE 1640394661 S[N] 4F034212-4E1A-DBFB-2931-16B0C1BC201C
+baike.ngotcm.com FALSE / FALSE 1354590137 hd_sid VdOweY
+baike.ngotcm.com FALSE / FALSE 1354590137 hd_hid 0
+baike.ngotcm.com FALSE / FALSE 1354590137 hd_auth 26f7svM9QAlUmx%2BeJy3mCzIKr40LTymtQusaIABXnFE4kTrA
+www.ngotcm.com FALSE / FALSE 1354607317 vSAE_2132_onlineindex 1
+www.ngotcm.com FALSE / FALSE 1354968751 vSAE_2132_home_readfeed 1323432751
+www.ngotcm.com FALSE / FALSE 1356933736 vSAE_2132_auth 66c5ZB%2FWkxAkufF9eqM5Z4%2ByCVCbAkUCntcmQdSIqO2RhwKRW656DXT3bRJL8RSDZq6Ro9XgcTcOvvyBZ%2BDl7dLBGTg
+www.ngotcm.com FALSE / FALSE 1326179687 vSAE_2132_invite_auth 5925%2Cieu7ce
+www.ngotcm.com FALSE / FALSE 1328187253 vSAE_2132_visitedfid 65D80D180D110D204D31D34D120D29D211D37
+.www.ngotcm.com TRUE / FALSE 1357131294 Hm_lvt_23234f06c4cd9a705eb19ee58a9d4470 1325595294100
+www.ngotcm.com FALSE / FALSE 1341320094 rtime 1
+www.ngotcm.com FALSE / FALSE 1341320094 ltime 1325595294130
+www.ngotcm.com FALSE / FALSE 1341320094 cnzz_eid 68370911-1323087412-http%3A//www.ngotcm.com/forum/forum-forumdisplay-fid-5.html
+www.ngotcm.com FALSE / FALSE 1357131360 vSAE_2132_smile 1D1
+www.ngotcm.com FALSE / FALSE 1328267802 vSAE_2132_lastvisit 1325672202
+www.ngotcm.com FALSE / FALSE 1325762202 vSAE_2132_sid NBVJLB
+www.ngotcm.com FALSE / FALSE 1325762210 vSAE_2132_lastact 1325675810%09home.php%09misc
Oops, something went wrong.

0 comments on commit aa8ecb4

Please sign in to comment.