TLS session cache not working with CURLOPT_TCP_FASTOPEN

[patrickkh7788]

When there is multi domain add to multi handle with CURLOPT_TCP_FASTOPEN, TLS cache not working.

Correct me if I am wrong. CURLINFO_APPCONNECT_TIME_T time on same domain seconds request >0, mean the TLS session cache not working.

#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <string.h>
#include <uv.h>
#include <curl/curl.h>
 
uv_loop_t *loop;
CURLM *curl_handle;
uv_timer_t timeout;

char* getPeerIp(curl_socket_t fd, char tmp[64]) {
	struct sockaddr addr;
	int err;
	switch(1) default : {
	    socklen_t namelen = sizeof(addr);
	    err = getpeername(fd, &addr, &namelen);
        if( err ) {
            //printf("err0: %d\n", err);
            break;
        }
        err     = uv_ip4_name((struct sockaddr_in*) &addr, tmp, 64);
        if( err ) {
            printf("err1: %d\n", err);
            break;
        }
        int port = ntohs(((struct sockaddr_in*)&addr)->sin_port);
        sprintf(tmp+strlen(tmp), ":%d", port);
	}
    return err ? "" : tmp ;
}

const char * const gURLS[] = {
    "https://www.weibo.com",
    "https://www.baidu.com"
};
const int gSize = sizeof(gURLS)/sizeof(gURLS[0]);
int gIndex = 0;
int gCount = 0;
int gMax   = gSize * 2 ;
static void add_download(const char *url, int num);
void getDownload(){
    if( gIndex >= gSize ) {
        gIndex  = 0;
    }
    if( gCount++ < gMax ) {
        add_download(gURLS[gIndex++], gCount);
    }
}

typedef struct curl_context_s {
  uv_poll_t poll_handle;
  curl_socket_t sockfd;
} curl_context_t;
 
static curl_context_t* create_curl_context(curl_socket_t sockfd)
{
  curl_context_t *context;
 
  context = (curl_context_t *) malloc(sizeof(*context));
 
  context->sockfd = sockfd;
 
  uv_poll_init_socket(loop, &context->poll_handle, sockfd);
  context->poll_handle.data = context;
  char tmp[64];
  printf("create_curl_context(%p, fd=%d, ip=%s)\n", context, sockfd, getPeerIp(sockfd, tmp));
 
  return context;
}
 
static void curl_close_cb(uv_handle_t *handle)
{
  curl_context_t *context = (curl_context_t *) handle->data;
  // char tmp[64];
  // printf(" ====> curl_close_socket: fd=%d ip=%s\n", context->sockfd, getPeerIp(context->sockfd, tmp));
  free(context);
}
 
static void destroy_curl_context(curl_context_t* context)
{
  uv_close((uv_handle_t *) &context->poll_handle, curl_close_cb);
}
 
static void add_download(const char *url, int num)
{
  char filename[50];
  FILE *file;
  CURL *handle;

  snprintf(filename, 50, "%d.download", num);

  file = fopen(filename, "wb");
  if(!file) {
    fprintf(stderr, "Error opening %s\n", filename);
    return;
  }
 
  handle = curl_easy_init();
  curl_easy_setopt(handle, CURLOPT_WRITEDATA, file);
  curl_easy_setopt(handle, CURLOPT_PRIVATE, file);
  curl_easy_setopt(handle, CURLOPT_URL, url);

  curl_easy_setopt(handle, CURLOPT_REFERER, url);
  curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L);
  curl_easy_setopt(handle, CURLOPT_AUTOREFERER, 1L);
  curl_easy_setopt(handle, CURLOPT_HEADER, 0L);
  curl_easy_setopt(handle, CURLOPT_TIMEOUT, 9);
  curl_easy_setopt(handle, CURLOPT_DNS_SERVERS, "8.8.8.8");
  curl_easy_setopt(handle, CURLOPT_REDIR_PROTOCOLS, -1);

  curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 1L);
  curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 2L);
  curl_easy_setopt(handle, CURLOPT_VERBOSE, 0L);

  curl_easy_setopt(handle, CURLOPT_TCP_KEEPALIVE, 1L);
  curl_easy_setopt(handle, CURLOPT_TCP_KEEPIDLE, 30L);
  curl_easy_setopt(handle, CURLOPT_TCP_KEEPINTVL, 15L);
  curl_easy_setopt(handle, CURLOPT_TCP_FASTOPEN, 1L);
  curl_multi_add_handle(curl_handle, handle);
  fprintf(stderr, "fetch(count=%d) %s\n", gCount, url);
}
 
static void check_multi_info(void)
{
  char *done_url;
  CURLMsg *message;
  int pending;
  CURL *easy_handle;
  FILE *file;
 
  while((message = curl_multi_info_read(curl_handle, &pending))) {
    switch(message->msg) {
    case CURLMSG_DONE:
      /* Do not use message data after calling curl_multi_remove_handle() and
         curl_easy_cleanup(). As per curl_multi_info_read() docs:
         "WARNING: The data the returned pointer points to will not survive
         calling curl_multi_cleanup, curl_multi_remove_handle or
         curl_easy_cleanup." */ 
      easy_handle = message->easy_handle;
 
      curl_easy_getinfo(easy_handle, CURLINFO_EFFECTIVE_URL, &done_url);
      curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &file);

      char *ip;
      curl_easy_getinfo(easy_handle, CURLINFO_PRIMARY_IP, &ip);
      long port;
      curl_easy_getinfo(easy_handle, CURLINFO_PRIMARY_PORT, &port);
      long ssltime;
      curl_easy_getinfo(easy_handle, CURLINFO_APPCONNECT_TIME_T, &ssltime);
      long conntime;
      curl_easy_getinfo(easy_handle, CURLINFO_CONNECT_TIME_T, &conntime);
      long dnstime;
      curl_easy_getinfo(easy_handle, CURLINFO_NAMELOOKUP_TIME_T, &dnstime);
      long time;
      curl_easy_getinfo(easy_handle, CURLINFO_TOTAL_TIME_T, &time);

	  curl_socket_t v;
      curl_easy_getinfo(easy_handle, CURLINFO_ACTIVESOCKET, &v);
      char tmp[64];
      printf("CURLMSG_DONE(count=%d): ip:%s:%ld fd=%d,%s conn=%ld dns=%ld ssl=%ld total=%ld \n\n",
        gCount,
        ip, port,
        v, getPeerIp(v, tmp),
       conntime/1000, dnstime/1000, ssltime / 1000, time/1000);
 
      curl_multi_remove_handle(curl_handle, easy_handle);
      curl_easy_cleanup(easy_handle);
      if(file) {
        fclose(file);
      }
      getDownload();
      break;
 
    default:
      fprintf(stderr, "CURLMSG default\n");
      break;
    }
  }
}
 
static void curl_perform(uv_poll_t *req, int status, int events)
{
  int running_handles;
  int flags = 0;
  curl_context_t *context;
 
  if(events & UV_READABLE)
    flags |= CURL_CSELECT_IN;
  if(events & UV_WRITABLE)
    flags |= CURL_CSELECT_OUT;
 
  context = (curl_context_t *) req->data;
 
  curl_multi_socket_action(curl_handle, context->sockfd, flags,
                           &running_handles);
 
  check_multi_info();
}
 
static void on_timeout(uv_timer_t *req)
{
  int running_handles;
  curl_multi_socket_action(curl_handle, CURL_SOCKET_TIMEOUT, 0,
                           &running_handles);
  check_multi_info();
}
 
static int start_timeout(CURLM *multi, long timeout_ms, void *userp)
{
  if(timeout_ms < 0) {
    uv_timer_stop(&timeout);
  }
  else {
    if(timeout_ms == 0)
      timeout_ms = 1; /* 0 means directly call socket_action, but we'll do it
                         in a bit */ 
    uv_timer_start(&timeout, on_timeout, timeout_ms, 0);
  }
  return 0;
}
 
static int handle_socket(CURL *easy, curl_socket_t s, int action, void *userp,
                  void *socketp)
{
  curl_context_t *curl_context;
  int events = 0;
  char tmp[64];
  switch(action) {
  case CURL_POLL_IN:
  case CURL_POLL_OUT:
  case CURL_POLL_INOUT:
    if( !socketp ) {
      // printf("action%d: fd=%d ip=%s\n", action, s, getPeerIp(s, tmp));
    }
    curl_context = socketp ?
      (curl_context_t *) socketp : create_curl_context(s);
 
    curl_multi_assign(curl_handle, s, (void *) curl_context);
 
    if(action != CURL_POLL_IN)
      events |= UV_WRITABLE;
    if(action != CURL_POLL_OUT)
      events |= UV_READABLE;
 
    uv_poll_start(&curl_context->poll_handle, events, curl_perform);
    break;
  case CURL_POLL_REMOVE:
    if(socketp) {
      printf("CURL_POLL_REMOVE: close(fd=%d, %s)\n", s, getPeerIp(s, tmp));
      uv_poll_stop(&((curl_context_t*)socketp)->poll_handle);
      destroy_curl_context((curl_context_t*) socketp);
      curl_multi_assign(curl_handle, s, NULL);
    } else {
        assert(socketp);
    }
    break;
  default:
    abort();
  }
 
  return 0;
}

int main(int argc, char **argv)
{
  loop = uv_default_loop();

  if(curl_global_init(CURL_GLOBAL_ALL)) {
    fprintf(stderr, "Could not init curl\n");
    return 1;
  }
 
  uv_timer_init(loop, &timeout);
 
  curl_handle = curl_multi_init();
  curl_multi_setopt(curl_handle, CURLMOPT_SOCKETFUNCTION, handle_socket);
  curl_multi_setopt(curl_handle, CURLMOPT_TIMERFUNCTION, start_timeout);
  curl_multi_setopt(curl_handle, CURLMOPT_MAXCONNECTS, 800L);
  curl_multi_setopt(curl_handle, CURLMOPT_MAX_HOST_CONNECTIONS, 200L);
  curl_multi_setopt(curl_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, 200L);

  getDownload();

  uv_run(loop, UV_RUN_DEFAULT);
  curl_multi_cleanup(curl_handle);
 
  return 0;
}

test output:

fetch(count=1) https://www.weibo.com
create_curl_context(0x7fcd65501140, fd=11, ip=114.114.114.114:53)
CURL_POLL_REMOVE: close(fd=11, 114.114.114.114:53)
create_curl_context(0x7fcd6550a0d0, fd=11, ip=)
CURL_POLL_REMOVE: close(fd=11, 180.149.134.141:443)
CURLMSG_DONE(count=1): ip::0 fd=11,180.149.134.141:443 conn=14 dns=14 ssl=113 total=147 

fetch(count=2) https://www.baidu.com
create_curl_context(0x7fcd65403bc0, fd=12, ip=114.114.114.114:53)
CURL_POLL_REMOVE: close(fd=12, 114.114.114.114:53)
create_curl_context(0x7fcd67808c80, fd=12, ip=)
CURL_POLL_REMOVE: close(fd=12, 180.101.49.12:443)
CURLMSG_DONE(count=2): ip::0 fd=12,180.101.49.12:443 conn=7 dns=7 ssl=37 total=47 

fetch(count=3) https://www.weibo.com
create_curl_context(0x7fcd65513020, fd=13, ip=)
CURL_POLL_REMOVE: close(fd=13, 180.149.134.141:443)
CURLMSG_DONE(count=3): ip::0 fd=13,180.149.134.141:443 conn=0 dns=0 ssl=87 total=116 

fetch(count=4) https://www.baidu.com
create_curl_context(0x7fcd6540c940, fd=14, ip=)
CURL_POLL_REMOVE: close(fd=14, 180.101.49.12:443)
CURLMSG_DONE(count=4): ip::0 fd=14,180.101.49.12:443 conn=0 dns=0 ssl=33 total=44 

remove CURLOPT_TCP_FASTOPEN, CURLINFO_APPCONNECT_TIME_T will be zero on the second request.

curl 7.65.3 with woflSSL v4.1.0, cares 1_15_0, confirm on macOS, linux, Android.