Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

initial commit

  • Loading branch information...
commit 95106385f918188458ff41197731231bbdaa91ba 0 parents
authored
83  README.md
Source Rendered
... ...
@@ -0,0 +1,83 @@
  1
+# curler
  2
+A native c++ node.js module for asynchronous http requests via libcurl.
  3
+
  4
+## Install
  5
+<pre>
  6
+  $ npm install curler
  7
+</pre>
  8
+
  9
+## request(options, callback[err, res, bodyData])
  10
+
  11
+### Options
  12
+ - `url`: request url. (required)
  13
+ - `method`: HTTP method type. Defaults to `GET`. (can be anything)
  14
+ - `headers`: Optional JSON key/value array of the request headers.
  15
+ - `data`: Optional request body data.
  16
+ - `timeout`: Total request timeout (connection/response) in milliseconds.
  17
+ - `connectionTimeout`: Connection timeout in milliseconds.
  18
+
  19
+## Examples
  20
+
  21
+### GET request
  22
+``` js
  23
+var curler = require("curler").Curler;
  24
+var curlClient = new curler();
  25
+
  26
+var options = {
  27
+  method: "GET",
  28
+  url: 'http://www.google.com'
  29
+};
  30
+
  31
+var startDate = Date.now();
  32
+curlClient.request(options, function(err, res, bodyData) {
  33
+  var duration = (Date.now() - startDate);
  34
+  if (err) {
  35
+    console.log(err);
  36
+  }
  37
+  else {
  38
+    console.log('statusCode: %s', res.statusCode);
  39
+    console.log('bodyData: %s', bodyData);
  40
+  }
  41
+
  42
+  console.log("curler (libcurl) performed http request in %s ms. dnsTime: %s, connectTime: %s, preTransferTime: %s, startTransferTime: %s, totalTime: %s", duration, res.dnsTime, res.connectTime, res.preTransferTime, res.startTransferTime, res.totalTime);
  43
+});
  44
+```
  45
+
  46
+### POST request (body data)
  47
+``` js
  48
+var curler = require("curler").Curler;
  49
+var curlClient = new curler();
  50
+
  51
+var data = JSON.stringify({ hello: 'world' });
  52
+
  53
+var options = {
  54
+  method: "POST",
  55
+  url: 'http://www.example.com/',
  56
+  headers: {
  57
+    'Content-Type': 'application/json',
  58
+    'Connection': 'Keep-Alive'
  59
+  },
  60
+  data: data,
  61
+  timeout: 5000,
  62
+  connectionTimeout: 5000
  63
+};
  64
+
  65
+var startDate = Date.now();
  66
+curlClient.request(options, function(err, res, bodyData) {
  67
+  var duration = (Date.now() - startDate);
  68
+  if (err) {
  69
+    console.log(err);
  70
+  }
  71
+  else {
  72
+    console.log('statusCode: %s', res.statusCode);
  73
+    console.log('bodyData: %s', bodyData);
  74
+  }
  75
+
  76
+  console.log("curler (libcurl) performed http request in %s ms. dnsTime: %s, connectTime: %s, preTransferTime: %s, startTransferTime: %s, totalTime: %s", duration, res.dnsTime, res.connectTime, res.preTransferTime, res.startTransferTime, res.totalTime);
  77
+});
  78
+```
  79
+
  80
+## TODO
  81
+- Proxy support (http/https)
  82
+- Allow Expect: 100-Continue to be configurable, rather than always off
  83
+- Load a queue of curl handles when the module loads (ghetto connection pooling). Need a deconstructor in curler.cc that works first!
22  package.json
... ...
@@ -0,0 +1,22 @@
  1
+{
  2
+		"name": "curler",
  3
+		"author": "Ryan French <frenchrya@gmail.com>",
  4
+		"version": "0.0.1",
  5
+		"description": "A native c++ node.js module for asynchronous http requests via libcurl.",
  6
+		"homepage": "http://github.com/rfrench/curler",
  7
+		"keywords": [
  8
+			"curl", 
  9
+			"libcurl",
  10
+			"http"
  11
+		],
  12
+		"repository": {
  13
+				"type": "git",
  14
+				"url": "git://github.com/rfrench/curler.git"
  15
+		},
  16
+		"main": "curler.node",
  17
+		"engines" : ["node >= 0.4.7"],
  18
+		"scripts": {
  19
+    "install" : "node-waf configure build install",
  20
+    "preuninstall": "rm -rf build/*"
  21
+  }
  22
+}
129  src/curl_client.cc
... ...
@@ -0,0 +1,129 @@
  1
+#define CURL_CLIENT
  2
+#include "curl_client.h"
  3
+
  4
+CurlClient::CurlClient() {}
  5
+
  6
+CurlClient::~CurlClient() {}
  7
+
  8
+curl_response CurlClient::Request(curl_request request) {
  9
+	curl_response response;
  10
+
  11
+	CURL *curl_handle = curl_easy_init();
  12
+ 
  13
+	/* set request URL */ 
  14
+	curl_easy_setopt(curl_handle, CURLOPT_URL, request.url.c_str());
  15
+
  16
+	/* set method type */
  17
+	if (request.method == "HEAD") {
  18
+		curl_easy_setopt(curl_handle, CURLOPT_NOBODY, 1); //no need to get response data if it's a HEAD request
  19
+	}
  20
+	else {
  21
+		curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, request.method.c_str());
  22
+	}
  23
+
  24
+	/* set timeout */
  25
+	if (request.timeout > 0) {
  26
+		curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, request.timeout);
  27
+	}
  28
+
  29
+	/* set connection timeout */
  30
+	if (request.connectionTimeout > 0) {
  31
+		curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT_MS, request.connectionTimeout);
  32
+	}
  33
+
  34
+	/* add custom headers */
  35
+	struct curl_slist *list = NULL;
  36
+	if (request.headers.size() > 0) {
  37
+		typedef std::map<string, string>::iterator it_type;
  38
+		for(it_type iterator = request.headers.begin(); iterator != request.headers.end(); iterator++) {
  39
+			string header = iterator->first + ": " + iterator->second;
  40
+			list = curl_slist_append(list, header.c_str());
  41
+		}
  42
+	}
  43
+	
  44
+	/* no expect header. not required but may be beneficial in some cases, but I don't need it. */
  45
+	/* todo: allow this to be configurable. */
  46
+	list = curl_slist_append(list, "Expect:");
  47
+	curl_easy_setopt (curl_handle, CURLOPT_HTTPHEADER, list);
  48
+
  49
+	/* send body data. Illegal to send body data with a GET request (even though libcurl will send it). */
  50
+	if (request.method != "GET") {
  51
+		if (request.bodyData.size() > 0) {
  52
+			curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, request.bodyData.c_str());
  53
+		}
  54
+	}
  55
+
  56
+	/* send all data to this function  */ 
  57
+	curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteCallback);
  58
+	curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &response);
  59
+
  60
+	/* we want the headers to this file handle */
  61
+	curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, HeaderCallback);
  62
+	curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &response.headers);
  63
+	
  64
+	/* set error buffer */
  65
+	char errorBuffer[CURL_ERROR_SIZE];
  66
+	curl_easy_setopt(curl_handle, CURLOPT_ERRORBUFFER, errorBuffer);
  67
+
  68
+	/* no progress meter please */ 
  69
+	curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1L);
  70
+
  71
+	/* todo: proxy support */
  72
+	/* do not verify peer or host */
  73
+	curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 0L);
  74
+	curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L);
  75
+
  76
+	/* let's do it */
  77
+	CURLcode res = curl_easy_perform(curl_handle);
  78
+	if (res != 0) {
  79
+		response.error = errorBuffer;
  80
+	}
  81
+
  82
+	/* lets get some stats on the request */
  83
+	curl_easy_getinfo(curl_handle, CURLINFO_TOTAL_TIME, &response.totalTime); 
  84
+	curl_easy_getinfo(curl_handle, CURLINFO_NAMELOOKUP_TIME, &response.dnsTime); 
  85
+	curl_easy_getinfo(curl_handle, CURLINFO_PRETRANSFER_TIME, &response.preTransferTime); 
  86
+	curl_easy_getinfo(curl_handle, CURLINFO_CONNECT_TIME, &response.connectTime); 
  87
+	curl_easy_getinfo(curl_handle, CURLINFO_STARTTRANSFER_TIME, &response.startTransferTime); 
  88
+
  89
+	/* get status code */
  90
+	response.statusCode = 0;
  91
+	curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &response.statusCode);
  92
+
  93
+	/* cleanup curl handle */ 
  94
+	curl_easy_cleanup(curl_handle);
  95
+
  96
+	return response;
  97
+}
  98
+
  99
+size_t CurlClient::HeaderCallback(void *buffer, size_t size, size_t nmemb, void *userp) {
  100
+	int result = size * nmemb;
  101
+
  102
+	std::string data = (char*) buffer;
  103
+	std::string key, value;
  104
+
  105
+	size_t p = data.find(":");
  106
+
  107
+	if((int)p > 0) {
  108
+		key = data.substr(0, p);
  109
+		value = data.substr(p + 2, (result - key.size() - 4)); //4 = 2 (aka ": " between header name and value) + carriage return 
  110
+		
  111
+		map<string, string> *headers = (map<string, string>*)userp;
  112
+		headers->insert(pair<string,string>(key, value));
  113
+	}
  114
+
  115
+	return result;
  116
+}
  117
+
  118
+size_t CurlClient::WriteCallback(void *contents, size_t size, size_t nmemb, void *userp) {
  119
+	int result = size * nmemb;
  120
+
  121
+	if (result > 0) {
  122
+		curl_response *response = (curl_response*)userp;
  123
+		string data = string((char*)contents, result);
  124
+
  125
+		response->data = response->data + data;
  126
+	}
  127
+
  128
+	return result;
  129
+}
43  src/curl_client.h
... ...
@@ -0,0 +1,43 @@
  1
+#ifndef CURLCLIENT_H
  2
+#define CURLCLIENT_H
  3
+#include <list>
  4
+#include <string>
  5
+#include <map>
  6
+#include <iostream>
  7
+#include <curl/curl.h>
  8
+#include <curl/easy.h>
  9
+
  10
+using namespace std;
  11
+
  12
+typedef struct {
  13
+	string method;
  14
+	string url;
  15
+	map<string, string> headers;
  16
+	string bodyData;
  17
+	long timeout;
  18
+	long connectionTimeout;
  19
+} curl_request;
  20
+
  21
+typedef struct {
  22
+	int statusCode;
  23
+	map<string, string> headers;
  24
+	string data;
  25
+	string error;
  26
+	double dnsTime;
  27
+	double connectTime;
  28
+	double preTransferTime;
  29
+	double startTransferTime;
  30
+	double totalTime;
  31
+} curl_response;
  32
+
  33
+class CurlClient {
  34
+ public:
  35
+	curl_response Request(curl_request request);
  36
+ ~CurlClient();
  37
+	CurlClient();
  38
+
  39
+ private:
  40
+	static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp);
  41
+	static size_t HeaderCallback(void *buffer, size_t size, size_t nmemb, void *userp);
  42
+};
  43
+#endif
221  src/curler.cc
... ...
@@ -0,0 +1,221 @@
  1
+#include <string>
  2
+#include <map>
  3
+#include <v8.h>
  4
+#include <node.h>
  5
+#include <curl/curl.h>
  6
+#include "curl_client.h"
  7
+
  8
+using namespace std;
  9
+using namespace node;
  10
+using namespace v8;
  11
+		
  12
+class Curler: ObjectWrap
  13
+{
  14
+	public:
  15
+		static Persistent<FunctionTemplate> s_ct;
  16
+		static void Init(Handle<Object> target)
  17
+		{
  18
+			Local<FunctionTemplate> t = FunctionTemplate::New(New);
  19
+
  20
+			s_ct = Persistent<FunctionTemplate>::New(t);
  21
+			s_ct->InstanceTemplate()->SetInternalFieldCount(1);
  22
+			s_ct->SetClassName(String::NewSymbol("Curler"));
  23
+
  24
+			NODE_SET_PROTOTYPE_METHOD(s_ct, "request", Request);
  25
+
  26
+			target->Set(String::NewSymbol("Curler"), s_ct->GetFunction());
  27
+		}
  28
+
  29
+		Curler() {}
  30
+		
  31
+		~Curler() {}
  32
+
  33
+		static Handle<Value> New(const Arguments& args)
  34
+		{
  35
+			Curler* hw = new Curler();
  36
+			hw->Wrap(args.This());
  37
+
  38
+			return args.This();
  39
+		}
  40
+
  41
+		/* struct for passing our request data around libeio */
  42
+		struct request_thread_data {
  43
+			Persistent<Function> cb;
  44
+			curl_request request;
  45
+			curl_response response;
  46
+		};
  47
+
  48
+		/**************************************************************************//**
  49
+		* HTTP request
  50
+		*
  51
+		* @args request options
  52
+		* @args callback to javascript
  53
+		*****************************************************************************/
  54
+		static Handle<Value> Request (const Arguments& args) {
  55
+			HandleScope scope;
  56
+			
  57
+			if (args.Length() != 2 || !args[0]->IsObject() || !args[1]->IsFunction()) {
  58
+			 	return ThrowException(Exception::TypeError(String::New("Usage: options, callback")));
  59
+			}
  60
+
  61
+			Local<Object> options = args[0]->ToObject();
  62
+			Local<Function> cb = Local<Function>::Cast(args[1]);
  63
+
  64
+			//create the request struct
  65
+			request_thread_data *rtd = new request_thread_data();
  66
+
  67
+			//create the curl request struct
  68
+			curl_request request;
  69
+
  70
+			//validate url
  71
+			if (!options->Has(String::New("url")) || options->Get(String::New("url"))->IsUndefined() || options->Get(String::New("url"))->IsNull()) { 
  72
+				return ThrowException(Exception::TypeError(String::New("URL is a required parameter."))); 
  73
+			}
  74
+			
  75
+			//set url
  76
+			request.url = *String::Utf8Value(options->Get(String::New("url")));
  77
+
  78
+			//set method type
  79
+			request.method = "GET";
  80
+			if (options->Has(String::New("method"))) {
  81
+				request.method = ToUpper(*String::Utf8Value(options->Get(String::New("method"))));
  82
+			}
  83
+
  84
+			//headers
  85
+			if (HasField(options, "headers")) {
  86
+				Local<Object> headers = options->Get(String::New("headers"))->ToObject();
  87
+				if (headers->IsObject()) {
  88
+					Local<Array> keys = headers->GetPropertyNames();
  89
+					for (size_t i = 0; i< keys->Length (); ++i) {
  90
+						Local<Value> key = keys->Get(i);
  91
+						Local<Value> value = headers->Get(key);
  92
+
  93
+						//add header
  94
+						request.headers.insert(pair<string,string>(*String::Utf8Value(key), *String::Utf8Value(value)));
  95
+					}
  96
+				}
  97
+			}
  98
+
  99
+			//body data
  100
+			if (options->Has(String::New("data"))) {
  101
+				if (options->Get(String::New("data"))->IsObject()) {
  102
+					return ThrowException(Exception::TypeError(String::New("bodyData cannot be an object. Try using JSON.stringify(data)."))); 
  103
+				}
  104
+
  105
+				request.bodyData = *String::Utf8Value(options->Get(String::New("data")));
  106
+			}
  107
+
  108
+			//timeout
  109
+			request.timeout = 0;
  110
+			if (options->Has(String::New("timeout"))) {
  111
+				request.timeout = options->Get(String::New("timeout"))->IntegerValue();
  112
+			}
  113
+
  114
+			//connection timeout
  115
+			request.connectionTimeout = 0;
  116
+			if (options->Has(String::New("connectionTimeout"))) {
  117
+				request.connectionTimeout = options->Get(String::New("connectionTimeout"))->IntegerValue();
  118
+			}
  119
+
  120
+			//set curl request data
  121
+			rtd->request = request;
  122
+
  123
+			//set callback function
  124
+			rtd->cb = Persistent<Function>::New(cb);
  125
+
  126
+			eio_custom((void (*)(eio_req*))RequestWorker, EIO_PRI_DEFAULT, RequestComplete, rtd);
  127
+			ev_ref(EV_DEFAULT_UC);
  128
+
  129
+			return scope.Close(Undefined());
  130
+		}
  131
+
  132
+		static int RequestWorker (eio_req *req) {
  133
+			request_thread_data *rtd = static_cast<request_thread_data *> (req->data);
  134
+
  135
+			CurlClient::CurlClient* curlClient = new CurlClient::CurlClient();
  136
+			rtd->response = curlClient->Request(rtd->request);
  137
+			delete curlClient;
  138
+
  139
+			return 0;
  140
+		}
  141
+
  142
+		/**************************************************************************//**
  143
+		* Request eio completion callback.
  144
+		*
  145
+		* @param request_thread_data
  146
+		*****************************************************************************/
  147
+		static int RequestComplete (eio_req *req) {
  148
+			request_thread_data *rtd = static_cast<request_thread_data *> (req->data);
  149
+			ev_unref(EV_DEFAULT_UC);
  150
+
  151
+			Local<Value> argv[3];
  152
+			argv[0] = Local<Value>::New(Null());
  153
+			if(!rtd->response.error.empty()) {
  154
+				argv[0] = String::New(rtd->response.error.c_str());
  155
+			}
  156
+
  157
+			//setup res object
  158
+			Local<Object> result = Object::New();
  159
+			result->Set(String::NewSymbol("statusCode"), Number::New(rtd->response.statusCode));
  160
+			
  161
+			//add headers
  162
+			Local<Object> headers = Object::New();
  163
+			typedef std::map<string, string>::iterator it_type;
  164
+			for(it_type iterator = rtd->response.headers.begin(); iterator != rtd->response.headers.end(); iterator++) {
  165
+				headers->Set(String::NewSymbol(ToLower(iterator->first).c_str()), String::New(iterator->second.c_str()));
  166
+			}
  167
+			result->Set(String::NewSymbol("headers"), headers);
  168
+
  169
+			//add perf numbers
  170
+			result->Set(String::NewSymbol("dnsTime"), Number::New(rtd->response.dnsTime));
  171
+			result->Set(String::NewSymbol("connectTime"), Number::New(rtd->response.connectTime));
  172
+			result->Set(String::NewSymbol("preTransferTime"), Number::New(rtd->response.preTransferTime));
  173
+			result->Set(String::NewSymbol("startTransferTime"), Number::New(rtd->response.startTransferTime));
  174
+			result->Set(String::NewSymbol("totalTime"), Number::New(rtd->response.totalTime));
  175
+			argv[1] = result;
  176
+
  177
+			//response data
  178
+			argv[2] = String::New(rtd->response.data.c_str());
  179
+
  180
+			/* Pass the argv array object to our callback function */
  181
+			TryCatch try_catch;
  182
+			rtd->cb->Call(Context::GetCurrent()->Global(), 3, argv);
  183
+			if (try_catch.HasCaught()) {
  184
+			 	FatalException(try_catch);
  185
+			}
  186
+			
  187
+			//peanut butter clean up time
  188
+			rtd->cb.Dispose();
  189
+			delete rtd;
  190
+
  191
+			return 0;
  192
+		}
  193
+	private:
  194
+		static bool HasField(Handle<Object> source, const char* fieldName)  {
  195
+			Local<String> field = String::New(fieldName);
  196
+
  197
+			return (source->Has (field) && !source->Get(field)->IsUndefined() && !source->Get(field)->IsNull());
  198
+		}
  199
+
  200
+		static string ToUpper(string data) {
  201
+			std::transform(data.begin(), data.end(), data.begin(), ::toupper);
  202
+			return data;
  203
+		}
  204
+
  205
+		static string ToLower(string data) {
  206
+			std::transform(data.begin(), data.end(), data.begin(), ::tolower);
  207
+			return data;
  208
+		}
  209
+};
  210
+
  211
+Persistent<FunctionTemplate> Curler::s_ct;
  212
+
  213
+extern "C" {
  214
+	static void init (Handle<Object> target)
  215
+	{
  216
+		eio_set_min_parallel(25);
  217
+		Curler::Init(target);
  218
+	}
  219
+
  220
+	NODE_MODULE(curler, init);
  221
+}
20  test.js
... ...
@@ -0,0 +1,20 @@
  1
+var curler = require("./build/Release/curler.node").Curler;
  2
+var curlClient = new curler();
  3
+
  4
+var options = {
  5
+	method: "GET",
  6
+	url: 'http://www.google.com'
  7
+};
  8
+
  9
+var startDate = Date.now();
  10
+curlClient.request(options, function(err, res, bodyData) {
  11
+	var duration = (Date.now() - startDate);
  12
+	if (err) {
  13
+		console.log(err);
  14
+	}
  15
+	else {
  16
+		console.log('statusCode: %s', res.statusCode);
  17
+		console.log('bodyData: %s', bodyData);
  18
+	}
  19
+	console.log("curler (libcurl) performed http request in %s ms. dnsTime: %s, connectTime: %s, preTransferTime: %s, startTransferTime: %s, totalTime: %s", duration, res.dnsTime, res.connectTime, res.preTransferTime, res.startTransferTime, res.totalTime);
  20
+});
21  wscript
... ...
@@ -0,0 +1,21 @@
  1
+import Options
  2
+from os import unlink, symlink, popen
  3
+from os.path import exists 
  4
+
  5
+srcdir = '.'
  6
+blddir = 'build'
  7
+VERSION = '0.0.1'
  8
+
  9
+def set_options(opt):
  10
+  opt.tool_options('compiler_cxx')
  11
+
  12
+def configure(conf):
  13
+  conf.check_tool('compiler_cxx')
  14
+  conf.check_tool('node_addon')
  15
+  conf.check(lib='curl')
  16
+
  17
+def build(bld):
  18
+  obj = bld.new_task_gen('cxx', 'shlib', 'node_addon')
  19
+  obj.target = 'curler'
  20
+  obj.source = 'src/curler.cc src/curl_client.cc'
  21
+  obj.lib = ['curl']

0 notes on commit 9510638

Please sign in to comment.
Something went wrong with that request. Please try again.