Skip to content
Permalink
Browse files

Add HTTP client cache.

This cache manages multiple http clients and returns one to the caller that is not busy.  It is the responsibility of the caller to indicate when they are done with a client. If returnContent is set then the client will automatically be marked done.

Also add special handing for HEAD requests to recognize that content-length is informational only and no content is expected.
  • Loading branch information...
dwsteele committed Jun 11, 2019
1 parent 6e809e5 commit ced42d6511e9b5735a2281dd0e1faec41870fd37
@@ -80,6 +80,7 @@ SRCS = \
common/io/filter/size.c \
common/io/handleRead.c \
common/io/handleWrite.c \
common/io/http/cache.c \
common/io/http/client.c \
common/io/http/common.c \
common/io/http/header.c \
@@ -306,6 +307,9 @@ common/io/handleRead.o: common/io/handleRead.c build.auto.h common/assert.h comm
common/io/handleWrite.o: common/io/handleWrite.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/io/handleWrite.c -o common/io/handleWrite.o

common/io/http/cache.o: common/io/http/cache.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/cache.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/io/http/cache.c -o common/io/http/cache.o

common/io/http/client.o: common/io/http/client.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/common.h common/io/http/header.h common/io/http/query.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/tls/client.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/io/http/client.c -o common/io/http/client.o

@@ -0,0 +1,106 @@
/***********************************************************************************************************************************
Http Client Cache
***********************************************************************************************************************************/
#include "build.auto.h"

#include "common/debug.h"
#include "common/io/http/cache.h"
#include "common/log.h"
#include "common/object.h"
#include "common/type/list.h"

/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct HttpClientCache
{
MemContext *memContext; // Mem context

const String *host; // Client settings
unsigned int port;
TimeMSec timeout;
bool verifyPeer;
const String *caFile;
const String *caPath;

List *clientList; // List of http clients
};

OBJECT_DEFINE_FREE(HTTP_CLIENT_CACHE);

/***********************************************************************************************************************************
New object
***********************************************************************************************************************************/
HttpClientCache *
httpClientCacheNew(
const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
{
FUNCTION_LOG_BEGIN(logLevelDebug)
FUNCTION_LOG_PARAM(STRING, host);
FUNCTION_LOG_PARAM(UINT, port);
FUNCTION_LOG_PARAM(TIME_MSEC, timeout);
FUNCTION_LOG_PARAM(BOOL, verifyPeer);
FUNCTION_LOG_PARAM(STRING, caFile);
FUNCTION_LOG_PARAM(STRING, caPath);
FUNCTION_LOG_END();

ASSERT(host != NULL);

HttpClientCache *this = NULL;

MEM_CONTEXT_NEW_BEGIN("HttpClientCache")
{
// Allocate state and set context
this = memNew(sizeof(HttpClientCache));
this->memContext = MEM_CONTEXT_NEW();

this->host = strDup(host);
this->port = port;
this->timeout = timeout;
this->verifyPeer = verifyPeer;
this->caFile = strDup(caFile);
this->caPath = strDup(caPath);

this->clientList = lstNew(sizeof(HttpClient *));
}
MEM_CONTEXT_NEW_END();

FUNCTION_LOG_RETURN(HTTP_CLIENT_CACHE, this);
}

/***********************************************************************************************************************************
Get an http client from the cache
***********************************************************************************************************************************/
HttpClient *
httpClientCacheGet(HttpClientCache *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace)
FUNCTION_LOG_PARAM(HTTP_CLIENT_CACHE, this);
FUNCTION_LOG_END();

ASSERT(this != NULL);

HttpClient *result = NULL;

// Search for a client that is not busy
for (unsigned int clientIdx = 0; clientIdx < lstSize(this->clientList); clientIdx++)
{
HttpClient *httpClient = *(HttpClient **)lstGet(this->clientList, clientIdx);

if (!httpClientBusy(httpClient))
result = httpClient;
}

// If none found then create a new one
if (result == NULL)
{
MEM_CONTEXT_BEGIN(this->memContext)
{
result = httpClientNew(this->host, this->port, this->timeout, this->verifyPeer, this->caFile, this->caPath);
lstAdd(this->clientList, &result);
}
MEM_CONTEXT_END();
}

FUNCTION_LOG_RETURN(HTTP_CLIENT, result);
}
@@ -0,0 +1,43 @@
/***********************************************************************************************************************************
Http Client Cache
Cache http clients and return one that is not busy on request.
***********************************************************************************************************************************/
#ifndef COMMON_IO_HTTP_CLIENT_CACHE_H
#define COMMON_IO_HTTP_CLIENT_CACHE_H

/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
#define HTTP_CLIENT_CACHE_TYPE HttpClientCache
#define HTTP_CLIENT_CACHE_PREFIX httpClientCache

typedef struct HttpClientCache HttpClientCache;

#include "common/io/http/client.h"

/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
HttpClientCache *httpClientCacheNew(
const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);

/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
HttpClient *httpClientCacheGet(HttpClientCache *this);

/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void httpClientCacheFree(HttpClientCache *this);

/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
#define FUNCTION_LOG_HTTP_CLIENT_CACHE_TYPE \
HttpClientCache *
#define FUNCTION_LOG_HTTP_CLIENT_CACHE_FORMAT(value, buffer, bufferSize) \
objToLog(value, "HttpClientCache", buffer, bufferSize)

#endif
@@ -119,6 +119,7 @@ httpClientRead(THIS_VOID, Buffer *buffer, bool block)
if (bufRemains(buffer) > this->contentRemaining)
bufLimitSet(buffer, bufSize(buffer) - (bufRemains(buffer) - (size_t)this->contentRemaining));

actualBytes = bufRemains(buffer);
this->contentRemaining -= ioRead(tlsClientIoRead(this->tls), buffer);

// Error if EOF but content read is not complete
@@ -243,8 +244,7 @@ httpClientRequest(
retry = false;

// Free the read interface
ioReadFree(this->ioRead);
this->ioRead = NULL;
httpClientDone(this);

// Free response status left over from the last request
httpHeaderFree(this->responseHeader);
@@ -386,7 +386,7 @@ httpClientRequest(
}

// Was content returned in the response?
bool contentExists = this->contentChunked || this->contentSize > 0;
bool contentExists = this->contentChunked || (this->contentSize > 0 && !strEq(verb, HTTP_VERB_HEAD_STR));
this->contentEof = !contentExists;

// If all content should be returned from this function then read the buffer. Also read the reponse if there has
@@ -482,6 +482,45 @@ httpClientStatStr(void)
FUNCTION_TEST_RETURN(result);
}

/***********************************************************************************************************************************
Mark the client as done if read is complete
***********************************************************************************************************************************/
void
httpClientDone(HttpClient *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(HTTP_CLIENT, this);
FUNCTION_LOG_END();

ASSERT(this != NULL);

if (this->ioRead != NULL)
{
if (!this->contentEof)
tlsClientClose(this->tls);

ioReadFree(this->ioRead);
this->ioRead = NULL;
}

FUNCTION_LOG_RETURN_VOID();
}

/***********************************************************************************************************************************
Is the http object busy?
***********************************************************************************************************************************/
bool
httpClientBusy(const HttpClient *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_CLIENT, this);
FUNCTION_TEST_END();

ASSERT(this != NULL);

FUNCTION_TEST_RETURN(this->ioRead);
}

/***********************************************************************************************************************************
Get read interface
***********************************************************************************************************************************/
@@ -70,6 +70,7 @@ HttpClient *httpClientNew(
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void httpClientDone(HttpClient *this);
Buffer *httpClientRequest(
HttpClient *this, const String *verb, const String *uri, const HttpQuery *query, const HttpHeader *requestHeader,
const Buffer *body, bool returnContent);
@@ -78,6 +79,7 @@ String *httpClientStatStr(void);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
bool httpClientBusy(const HttpClient *this);
IoRead *httpClientIoRead(const HttpClient *this);
unsigned int httpClientResponseCode(const HttpClient *this);
const HttpHeader *httpClientReponseHeader(const HttpClient *this);
@@ -230,9 +230,10 @@ unit:

# ----------------------------------------------------------------------------------------------------------------------------
- name: io-http
total: 4
total: 5

coverage:
common/io/http/cache: full
common/io/http/client: full
common/io/http/common: full
common/io/http/header: full
@@ -142,6 +142,16 @@ testHttpServer(void)
"Connection:ack\r\n"
"\r\n");

// Head request with no content-length but no content
harnessTlsServerExpect(
"HEAD / HTTP/1.1\r\n"
"\r\n");

harnessTlsServerReply(
"HTTP/1.1 200 OK\r\n"
"content-length:380\r\n"
"\r\n");

// Error with content length 0 (with a few slow down errors)
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
@@ -453,6 +463,17 @@ testRun(void)
strPtr(httpHeaderToLog(httpClientReponseHeader(client))), "{connection: 'ack', key1: '0', key2: 'value2'}",
" check response headers");

// Head request with no content-length but no content
TEST_RESULT_VOID(
httpClientRequest(client, strNew("HEAD"), strNew("/"), NULL, httpHeaderNew(NULL), NULL, true),
"head request with content-length");
TEST_RESULT_UINT(httpClientResponseCode(client), 200, " check response code");
TEST_RESULT_STR(strPtr(httpClientResponseMessage(client)), "OK", " check response message");
TEST_RESULT_BOOL(httpClientEof(client), true, " io is eof");
TEST_RESULT_BOOL(httpClientBusy(client), false, " client is not busy");
TEST_RESULT_STR(
strPtr(httpHeaderToLog(httpClientReponseHeader(client))), "{content-length: '380'}", " check response headers");

// Error with content length 0
TEST_RESULT_VOID(
httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "error with content length 0");
@@ -500,6 +521,7 @@ testRun(void)
TEST_RESULT_VOID(
httpClientRequest(client, strNew("GET"), strNew("/path/file 1.txt"), NULL, NULL, NULL, false),
"request with content length error");
TEST_RESULT_BOOL(httpClientBusy(client), true, " client is busy");
TEST_ERROR(
ioRead(httpClientIoRead(client), buffer), FileReadError, "unexpected EOF reading HTTP content");

@@ -519,5 +541,30 @@ testRun(void)
TEST_RESULT_VOID(httpClientFree(client), "free client");
}

// *****************************************************************************************************************************
if (testBegin("HttpClientCache"))
{
HttpClientCache *cache = NULL;
HttpClient *client1 = NULL;
HttpClient *client2 = NULL;

TEST_ASSIGN(cache, httpClientCacheNew(strNew("localhost"), TLS_TEST_PORT, 500, true, NULL, NULL), "new http client cache");
TEST_ASSIGN(client1, httpClientCacheGet(cache), "get http client");
TEST_RESULT_PTR(client1, *(HttpClient **)lstGet(cache->clientList, 0), " check http client");
TEST_RESULT_PTR(httpClientCacheGet(cache), *(HttpClient **)lstGet(cache->clientList, 0), " get same http client");

// Make client 1 look like it is busy
client1->ioRead = (IoRead *)1;

TEST_ASSIGN(client2, httpClientCacheGet(cache), "get http client");
TEST_RESULT_PTR(client2, *(HttpClient **)lstGet(cache->clientList, 1), " check http client");
TEST_RESULT_BOOL(client1 != client2, true, "clients are not the same");

// Set back to NULL so bad things don't happen during free
client1->ioRead = NULL;

TEST_RESULT_VOID(httpClientCacheFree(cache), "free http client cache");
}

FUNCTION_HARNESS_RESULT_VOID();
}

0 comments on commit ced42d6

Please sign in to comment.
You can’t perform that action at this time.