Skip to content

Commit

Permalink
always-multi: always use non-blocking internals
Browse files Browse the repository at this point in the history 
Remove internal separated behavior of the easy vs multi intercace.
curl_easy_perform() is now using the multi interface itself.

Several minor multi interface quirks and bugs have been fixed in the
process.

Much help with debugging this has been provided by: Yang Tse
  • Loading branch information
@bagder
bagder committed Jan 17, 2013
1 parent 9fd88ab commit c431274
Show file tree
Hide file tree
Showing 54 changed files with 489 additions and 883 deletions.
14 changes: 2 additions & 12 deletions docs/TODO
View file
Expand Up @@ -21,8 +21,7 @@

2. libcurl - multi interface
2.1 More non-blocking
2.2 Remove easy interface internally
2.4 Fix HTTP Pipelining for PUT
2.2 Fix HTTP Pipelining for PUT

3. Documentation
3.1 More and better
Expand Down Expand Up @@ -191,16 +190,7 @@
- The "DONE" operation (post transfer protocol-specific actions) for the
protocols SFTP, SMTP, FTP. Fixing Curl_done() for this is a worthy task.

2.2 Remove easy interface internally

Make curl_easy_perform() a wrapper-function that simply creates a multi
handle, adds the easy handle to it, runs curl_multi_perform() until the
transfer is done, then detach the easy handle, destroy the multi handle and
return the easy handle's return code. This will thus make everything
internally use and assume the multi interface. The select()-loop should use
curl_multi_socket().

2.4 Fix HTTP Pipelining for PUT
2.2 Fix HTTP Pipelining for PUT

HTTP Pipelining can be a way to greatly enhance performance for multiple
serial requests and currently libcurl only supports that for HEAD and GET
Expand Down
2 changes: 1 addition & 1 deletion lib/Makefile.inc
View file
Expand Up @@ -43,4 +43,4 @@ HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h \
gopher.h axtls.h cyassl.h http_proxy.h non-ascii.h asyn.h curl_ntlm.h \
curl_gssapi.h curl_ntlm_wb.h curl_ntlm_core.h curl_ntlm_msgs.h \
curl_sasl.h curl_schannel.h curl_multibyte.h curl_darwinssl.h \
hostcheck.h bundles.h conncache.h curl_setup_once.h
hostcheck.h bundles.h conncache.h curl_setup_once.h multihandle.h
7 changes: 2 additions & 5 deletions lib/conncache.c
View file
Expand Up @@ -6,7 +6,7 @@
* \___|\___/|_| \_\_____|
*
* Copyright (C) 2012, Linus Nielsen Feltzing, <linus@haxx.se>
* Copyright (C) 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
* Copyright (C) 2012 - 2013, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
Expand Down Expand Up @@ -47,7 +47,7 @@ static void free_bundle_hash_entry(void *freethis)
Curl_bundle_destroy(b);
}

struct conncache *Curl_conncache_init(conncachetype type)
struct conncache *Curl_conncache_init(void)
{
struct conncache *connc;

Expand All @@ -63,9 +63,6 @@ struct conncache *Curl_conncache_init(conncachetype type)
return NULL;
}

connc->type = type;
connc->num_connections = 0;

return connc;
}

Expand Down
10 changes: 2 additions & 8 deletions lib/conncache.h
View file
Expand Up @@ -7,7 +7,7 @@
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 2012, Linus Nielsen Feltzing, <linus@haxx.se>
* Copyright (C) 2012, 2013, Linus Nielsen Feltzing, <linus@haxx.se>
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
Expand All @@ -22,18 +22,12 @@
*
***************************************************************************/

typedef enum {
CONNCACHE_PRIVATE, /* used for an easy handle alone */
CONNCACHE_MULTI /* shared within a multi handle */
} conncachetype;

struct conncache {
struct curl_hash *hash;
conncachetype type;
size_t num_connections;
};

struct conncache *Curl_conncache_init(conncachetype type);
struct conncache *Curl_conncache_init(void);

void Curl_conncache_destroy(struct conncache *connc);

Expand Down
42 changes: 35 additions & 7 deletions lib/connect.c
View file
Expand Up @@ -5,7 +5,7 @@
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
* Copyright (C) 1998 - 2013, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
Expand Down Expand Up @@ -74,6 +74,8 @@
#include "sslgen.h" /* for Curl_ssl_check_cxn() */
#include "progress.h"
#include "warnless.h"
#include "conncache.h"
#include "multihandle.h"

/* The last #include file should be: */
#include "memdebug.h"
Expand Down Expand Up @@ -980,8 +982,7 @@ singleipconnect(struct connectdata *conn,

/* The 'WAITCONN_TIMEOUT == rc' comes from the waitconnect(), and not from
connect(). We can be sure of this since connect() cannot return 1. */
if((WAITCONN_TIMEOUT == rc) &&
(data->state.used_interface == Curl_if_multi)) {
if(WAITCONN_TIMEOUT == rc) {
/* Timeout when running the multi interface */
*sockp = sockfd;
return CURLE_OK;
Expand Down Expand Up @@ -1072,9 +1073,8 @@ CURLcode Curl_connecthost(struct connectdata *conn, /* context */

/* start connecting to the IP curr_addr points to */
res = singleipconnect(conn, curr_addr,
/* don't hang when doing multi */
(data->state.used_interface == Curl_if_multi)?0:
conn->timeoutms_per_addr, &sockfd, connected);
0, /* don't hang when doing multi */
&sockfd, connected);
if(res)
return res;

Expand Down Expand Up @@ -1112,6 +1112,21 @@ CURLcode Curl_connecthost(struct connectdata *conn, /* context */
return CURLE_OK;
}

struct connfind {
struct connectdata *tofind;
bool found;
};

static int conn_is_conn(struct connectdata *conn, void *param)
{
struct connfind *f = (struct connfind *)param;
if(conn == f->tofind) {
f->found = TRUE;
return 1;
}
return 0;
}

/*
* Used to extract socket and connectdata struct for the most recent
* transfer on the given SessionHandle.
Expand All @@ -1125,8 +1140,21 @@ curl_socket_t Curl_getconnectinfo(struct SessionHandle *data,

DEBUGASSERT(data);

if(data->state.lastconnect) {
/* this only works for an easy handle that has been used for
curl_easy_perform()! */
if(data->state.lastconnect && data->multi_easy) {
struct connectdata *c = data->state.lastconnect;
struct connfind find;
find.tofind = data->state.lastconnect;
find.found = FALSE;

Curl_conncache_foreach(data->multi_easy->conn_cache, &find, conn_is_conn);

if(!find.found) {
data->state.lastconnect = NULL;
return CURL_SOCKET_BAD;
}

if(connp)
/* only store this if the caller cares for it */
*connp = c;
Expand Down
161 changes: 44 additions & 117 deletions lib/easy.c
View file
Expand Up @@ -5,7 +5,7 @@
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
* Copyright (C) 1998 - 2013, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
Expand Down Expand Up @@ -385,40 +385,46 @@ CURLcode curl_easy_setopt(CURL *curl, CURLoption tag, ...)
return ret;
}

#ifdef CURL_MULTIEASY
/***************************************************************************
* This function is still only for testing purposes. It makes a great way
* to run the full test suite on the multi interface instead of the easy one.
***************************************************************************
*
* The *new* curl_easy_perform() is the external interface that performs a
* transfer previously setup.
/*
* curl_easy_perform() is the external interface that performs a blocking
* transfer as previously setup.
*
* Wrapper-function that: creates a multi handle, adds the easy handle to it,
* CONCEPT: This function creates a multi handle, adds the easy handle to it,
* runs curl_multi_perform() until the transfer is done, then detaches the
* easy handle, destroys the multi handle and returns the easy handle's return
* code. This will make everything internally use and assume multi interface.
* code.
*
* REALITY: it can't just create and destroy the multi handle that easily. It
* needs to keep it around since if this easy handle is used again by this
* function, the same multi handle must be re-used so that the same pools and
* caches can be used.
*/
CURLcode curl_easy_perform(CURL *easy)
{
CURLM *multi;
CURLMcode mcode;
CURLcode code = CURLE_OK;
int still_running;
struct timeval timeout;
int rc;
CURLMsg *msg;
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd;
bool done = FALSE;
int rc;
struct SessionHandle *data = easy;

if(!easy)
return CURLE_BAD_FUNCTION_ARGUMENT;

multi = curl_multi_init();
if(!multi)
return CURLE_OUT_OF_MEMORY;
if(data->multi) {
failf(data, "easy handled already used in multi handle");
return CURLE_FAILED_INIT;
}

if(data->multi_easy)
multi = data->multi_easy;
else {
multi = curl_multi_init();
if(!multi)
return CURLE_OUT_OF_MEMORY;
data->multi_easy = multi;
}

mcode = curl_multi_add_handle(multi, easy);
if(mcode) {
Expand All @@ -429,108 +435,33 @@ CURLcode curl_easy_perform(CURL *easy)
return CURLE_FAILED_INIT;
}

/* we start some action by calling perform right away */

do {
while(CURLM_CALL_MULTI_PERFORM ==
curl_multi_perform(multi, &still_running));

if(!still_running)
break;

FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);

/* timeout once per second */
timeout.tv_sec = 1;
timeout.tv_usec = 0;

/* Old deprecated style: get file descriptors from the transfers */
curl_multi_fdset(multi, &fdread, &fdwrite, &fdexcep, &maxfd);
rc = Curl_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);

/* The way is to extract the sockets and wait for them without using
select. This whole alternative version should probably rather use the
curl_multi_socket() approach. */

if(rc == -1)
/* select error */
break;

/* timeout or data to send/receive => loop! */
} while(still_running);

msg = curl_multi_info_read(multi, &rc);
if(msg)
code = msg->data.result;

mcode = curl_multi_remove_handle(multi, easy);
/* what to do if it fails? */

mcode = curl_multi_cleanup(multi);
/* what to do if it fails? */

return code;
}
#else
/*
* curl_easy_perform() is the external interface that performs a transfer
* previously setup.
*/
CURLcode curl_easy_perform(CURL *curl)
{
struct SessionHandle *data = (struct SessionHandle *)curl;

if(!data)
return CURLE_BAD_FUNCTION_ARGUMENT;
/* assign this after curl_multi_add_handle() since that function checks for
it and rejects this handle otherwise */
data->multi = multi;

if(! (data->share && data->share->hostcache)) {
/* this handle is not using a shared dns cache */
while(!done && !mcode) {
int still_running;

if(data->set.global_dns_cache &&
(data->dns.hostcachetype != HCACHE_GLOBAL)) {
/* global dns cache was requested but still isn't */
struct curl_hash *ptr;
mcode = curl_multi_wait(multi, NULL, 0, 1000, NULL);

if(data->dns.hostcachetype == HCACHE_PRIVATE) {
/* if the current cache is private, kill it first */
Curl_hash_destroy(data->dns.hostcache);
data->dns.hostcachetype = HCACHE_NONE;
data->dns.hostcache = NULL;
}
if(mcode == CURLM_OK)
mcode = curl_multi_perform(multi, &still_running);

ptr = Curl_global_host_cache_init();
if(ptr) {
/* only do this if the global cache init works */
data->dns.hostcache = ptr;
data->dns.hostcachetype = HCACHE_GLOBAL;
/* only read 'still_running' if curl_multi_perform() return OK */
if((mcode == CURLM_OK) && !still_running) {
msg = curl_multi_info_read(multi, &rc);
if(msg) {
code = msg->data.result;
done = TRUE;
}
}

if(!data->dns.hostcache) {
data->dns.hostcachetype = HCACHE_PRIVATE;
data->dns.hostcache = Curl_mk_dnscache();

if(!data->dns.hostcache)
/* While we possibly could survive and do good without a host cache,
the fact that creating it failed indicates that things are truly
screwed up and we should bail out! */
return CURLE_OUT_OF_MEMORY;
}

}

if(!data->state.conn_cache) {
/* Oops, no connection cache, create one */
data->state.conn_cache = Curl_conncache_init(CONNCACHE_PRIVATE);
if(!data->state.conn_cache)
return CURLE_OUT_OF_MEMORY;
}
mcode = curl_multi_remove_handle(multi, easy);

return Curl_perform(data);
/* The multi handle is kept alive, owned by the easy handle */
return code;
}
#endif

/*
* curl_easy_cleanup() is the external interface to cleaning/freeing the given
Expand All @@ -553,10 +484,6 @@ void Curl_easy_addmulti(struct SessionHandle *data,
void *multi)
{
data->multi = multi;
if(multi == NULL)
/* the association is cleared, mark the easy handle as not used by an
interface */
data->state.used_interface = Curl_if_none;
}

void Curl_easy_initHandleData(struct SessionHandle *data)
Expand Down

0 comments on commit c431274

Please sign in to comment.