Permalink
Browse files

Initial import

  • Loading branch information...
0 parents commit 9fa2c1f52da26d25e3b99f8366780b09cd762b4e andris9 committed Oct 13, 2011
Showing with 235 additions and 0 deletions.
  1. +16 −0 LICENSE
  2. +23 −0 README.md
  3. +160 −0 fetch.js
  4. +29 −0 package.json
  5. +7 −0 test.js
16 LICENSE
@@ -0,0 +1,16 @@
+Copyright (c) 2011 Andris Reinman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
@@ -0,0 +1,23 @@
+# fetch
+
+Fetch url contents
+
+## Install
+
+ npm install fetch
+
+## Usage
+
+ var fetch = require("fetch");
+
+ fetch("http://www.google.com", function(error, response){
+ console.log(response.status);
+ console.log(response.headers);
+ console.log(response.body);
+ });
+
+See test.js for a complete example
+
+## License
+
+BSD
160 fetch.js
@@ -0,0 +1,160 @@
+var http = require("http"),
+ https = require("https"),
+ urllib = require("url"),
+ zlib = require('zlib');
+
+/*
+
+headers: {}
+payload: txt
+
+maxresponse (100kB)
+headers: {}
+maxredirects: 10
+
+*/
+
+module.exports = fetch;
+
+function fetch(url, options, callback){
+
+ if(!callback && typeof options == "function"){
+ callback = options;
+ options = undefined;
+ }
+
+ options = options || {};
+ if(typeof options.maxredirects != "number" && !(options.maxredirects instanceof Number)){
+ options.maxredirects = 10;
+ }
+
+ // todo: redirects
+ function go(url, i){
+ i = i || 0;
+
+ get(url, options, function(error, response){
+ if(error){
+ return callback(error);
+ }
+ if([301, 302].indexOf(response.status)>=0 && response.headers.location){
+ i++;
+ if(i>options.maxredirects){
+ return callback(null, response);
+ }
+ go(response.headers.location, i);
+ }else{
+ return callback(null, response);
+ }
+ });
+ }
+
+ go(url);
+
+}
+
+function get(url, options, callback){
+
+ if(!callback && typeof options == "function"){
+ callback = options;
+ options = undefined;
+ }
+
+ options = options || {};
+
+ options.maxresponse = options.maxresponse || (100*1024); // 100kB
+
+ var urlparts = urllib.parse(url, false, true),
+ transport,
+ urloptions = {
+ host: urlparts.hostname,
+ port: urlparts.port,
+ path: urlparts.pathname + (urlparts.search || "") || "/",
+ method: options.payload?'POST':'GET'
+ };
+
+ if(!urloptions.port){
+ switch(urlparts.protocol){
+ case "https:":
+ urloptions.port = 443;
+ transport = https;
+ break;
+ case "https:":
+ default:
+ urloptions.port = 80;
+ transport = http;
+ break;
+ }
+ }
+
+ if(options.headers){
+ urloptions.headers = options.headers;
+ }else{
+ urloptions.headers = {};
+ }
+
+ if(!options.nocompress){
+ urloptions.headers['Accept-Encoding'] = 'gzip';
+ }
+
+ var req = transport.request(urloptions, function(res) {
+
+ var responseBody = new Buffer(0),
+ currentPart,
+ maxlen,
+ unpack,
+
+ receive = function(chunk){
+
+ if(responseBody.length + chunk.length>options.maxresponse){
+ maxlen = options.maxresponse - responseBody.length;
+ }else{
+ maxlen = chunk.length;
+ }
+ if(maxlen<=0)return;
+
+ currentPart = new Buffer(responseBody.length + maxlen);
+ responseBody.copy(currentPart);
+ chunk.copy(currentPart, responseBody.length, 0, maxlen);
+ responseBody = currentPart;
+ },
+
+ end = function(){
+ callback(null, {
+ status: res.statusCode,
+ headers: res.headers,
+ body: responseBody
+ });
+ }
+
+ if(res.headers['content-encoding']){
+ switch(res.headers['content-encoding'].toLowerCase().trim()){
+ case "gzip":
+ unpack = zlib.createGunzip();
+ unpack.on("data", receive);
+ unpack.on("error", callback);
+ unpack.on("end", end);
+ res.pipe(unpack);
+ return;
+ case "deflate":
+ unpack = zlib.createInflateRaw();
+ unpack.on("data", receive);
+ unpack.on("error", callback);
+ unpack.on("end", end);
+ res.pipe(unpack);
+ return;
+ }
+ }
+
+ res.on('data', receive);
+ res.on('end', end);
+ });
+
+ req.on('error', callback);
+
+ if(options.payload){
+ req.end(options.payload);
+ }else{
+ req.end();
+ }
+
+}
@@ -0,0 +1,29 @@
+{
+ "name": "fetch",
+ "description": "Fetch URL contents",
+ "version": "0.1.0",
+ "author" : "Andris Reinman",
+ "maintainers":[
+ {
+ "name":"andris",
+ "email":"andris@node.ee"
+ }
+ ],
+ "homepage": "http://github.com/andris9/fetch",
+ "repository" : {
+ "type" : "git",
+ "url" : "http://github.com/andris9/fetch.git"
+ },
+ "main" : "./fetch",
+ "licenses" : [
+ {
+ "type": "MIT",
+ "url": "http://github.com/andris9/fetch/blob/master/LICENSE"
+ }
+ ],
+ "dependencies": {
+ "mimelib-noiconv":"*"
+ },
+ "engine": [ "node >=0.5.0" ],
+ "keywords": ["url"]
+}
@@ -0,0 +1,7 @@
+var fetch = require("./fetch");
+
+fetch("https://www.google.com", function(error, contents){
+ console.log(error || contents);
+ console.log(contents.body.toString("utf-8"))
+ console.log(contents.body.toString("utf-8").length)
+});

0 comments on commit 9fa2c1f

Please sign in to comment.