188 changes: 138 additions & 50 deletions htp/htp_decompressors.c
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@

#include "htp_private.h"


static void *SzAlloc(ISzAllocPtr p, size_t size) { return malloc(size); }
static void SzFree(ISzAllocPtr p, void *address) { free(address); }
const ISzAlloc lzma_Alloc = { SzAlloc, SzFree };


/**
* @brief See if the header has extensions
* @return number of bytes to skip
Expand Down Expand Up @@ -153,6 +159,21 @@ static int htp_gzip_decompressor_restart(htp_decompressor_gzip_t *drec,
return 1;
}

/**
* Ends decompressor.
*
* @param[in] drec
*/
static void htp_gzip_decompressor_end(htp_decompressor_gzip_t *drec) {
if (drec->zlib_initialized == HTP_COMPRESSION_LZMA) {
LzmaDec_Free(&drec->state, &lzma_Alloc);
drec->zlib_initialized = 0;
} else if (drec->zlib_initialized) {
inflateEnd(&drec->stream);
drec->zlib_initialized = 0;
}
}

/**
* Decompress a chunk of gzip-compressed data.
* If we have more than one decompressor, call this function recursively.
Expand All @@ -164,6 +185,7 @@ static int htp_gzip_decompressor_restart(htp_decompressor_gzip_t *drec,
static htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_gzip_t *drec, htp_tx_data_t *d) {
size_t consumed = 0;
int rc = 0;
htp_status_t callback_rc;

// Pass-through the NULL chunk, which indicates the end of the stream.

Expand All @@ -174,7 +196,7 @@ static htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_gzip_t *dr
d2.len = d->len;
d2.is_last = d->is_last;

htp_status_t callback_rc = drec->super.callback(&d2);
callback_rc = drec->super.callback(&d2);
if (callback_rc != HTP_OK) {
return HTP_ERROR;
}
Expand All @@ -186,18 +208,21 @@ static htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_gzip_t *dr
// Prepare data for callback.
htp_tx_data_t dout;
dout.tx = d->tx;
dout.data = NULL;
dout.len = 0;
// This is last call, so output uncompressed data so far
dout.len = GZIP_BUF_SIZE - drec->stream.avail_out;
if (dout.len > 0) {
dout.data = drec->buffer;
} else {
dout.data = NULL;
}
dout.is_last = d->is_last;
if (drec->super.next != NULL && drec->zlib_initialized) {
return htp_gzip_decompressor_decompress((htp_decompressor_gzip_t *)drec->super.next, &dout);
} else {
// Send decompressed data to the callback.
htp_status_t callback_rc = drec->super.callback(&dout);
callback_rc = drec->super.callback(&dout);
if (callback_rc != HTP_OK) {
inflateEnd(&drec->stream);
drec->zlib_initialized = 0;

htp_gzip_decompressor_end(drec);
return callback_rc;
}
}
Expand Down Expand Up @@ -226,31 +251,77 @@ static htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_gzip_t *dr
d2.len = GZIP_BUF_SIZE;
d2.is_last = d->is_last;

//if (drec->super.next != NULL) {
if (drec->super.next != NULL && drec->zlib_initialized) {
htp_tx_data_t d3;
d3.tx = d->tx;
d3.data = drec->buffer;
d3.len = GZIP_BUF_SIZE;
d3.is_last = d->is_last;
return htp_gzip_decompressor_decompress((htp_decompressor_gzip_t *)drec->super.next, &d3);
callback_rc = htp_gzip_decompressor_decompress((htp_decompressor_gzip_t *)drec->super.next, &d2);
} else {
// Send decompressed data to callback.
htp_status_t callback_rc = drec->super.callback(&d2);
if (callback_rc != HTP_OK) {
inflateEnd(&drec->stream);
drec->zlib_initialized = 0;

return callback_rc;
}
callback_rc = drec->super.callback(&d2);
}
if (callback_rc != HTP_OK) {
htp_gzip_decompressor_end(drec);
return callback_rc;
}

drec->stream.next_out = drec->buffer;
drec->stream.avail_out = GZIP_BUF_SIZE;
}

if (drec->zlib_initialized) {
if (drec->zlib_initialized == HTP_COMPRESSION_LZMA) {
if (drec->header_len < LZMA_PROPS_SIZE + 8) {
consumed = LZMA_PROPS_SIZE + 8 - drec->header_len;
if (consumed > drec->stream.avail_in) {
consumed = drec->stream.avail_in;
}
memcpy(drec->header + drec->header_len, drec->stream.next_in, consumed);
drec->stream.next_in = (unsigned char *) (d->data + consumed);
drec->stream.avail_in = d->len - consumed;
drec->header_len += consumed;
}
if (drec->header_len == LZMA_PROPS_SIZE + 8) {
rc = LzmaDec_Allocate(&drec->state, drec->header, LZMA_PROPS_SIZE, &lzma_Alloc);
if (rc != SZ_OK)
return rc;
LzmaDec_Init(&drec->state);
// hacky to get to next step end retry allocate in case of failure
drec->header_len++;
}
if (drec->header_len > LZMA_PROPS_SIZE + 8) {
size_t inprocessed = drec->stream.avail_in;
size_t outprocessed = drec->stream.avail_out;
ELzmaStatus status;
rc = LzmaDec_DecodeToBuf(&drec->state, drec->stream.next_out, &outprocessed,
drec->stream.next_in, &inprocessed, LZMA_FINISH_ANY, &status, d->tx->cfg->lzma_memlimit);
drec->stream.avail_in -= inprocessed;
drec->stream.next_in += inprocessed;
drec->stream.avail_out -= outprocessed;
drec->stream.next_out += outprocessed;
switch (rc) {
case SZ_OK:
rc = Z_OK;
if (status == LZMA_STATUS_FINISHED_WITH_MARK) {
rc = Z_STREAM_END;
}
break;
case SZ_ERROR_MEM:
htp_log(d->tx->connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "LZMA decompressor: memory limit reached");
// fall through
default:
rc = Z_DATA_ERROR;
}
}
} else if (drec->zlib_initialized) {
rc = inflate(&drec->stream, Z_NO_FLUSH);
} else {
// no initialization means previous error on stream
return HTP_ERROR;
}
if (GZIP_BUF_SIZE > drec->stream.avail_out) {
if (rc == Z_DATA_ERROR) {
// There is data even if there is an error
// So use this data and log a warning
htp_log(d->tx->connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "GZip decompressor: inflate failed with %d", rc);
rc = Z_STREAM_END;
}
}
if (rc == Z_STREAM_END) {
// How many bytes do we have?
Expand All @@ -266,30 +337,30 @@ static htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_gzip_t *dr
d2.is_last = d->is_last;

if (drec->super.next != NULL && drec->zlib_initialized) {
htp_tx_data_t d3;
d3.tx = d->tx;
d3.data = drec->buffer;
d3.len = len;
d3.is_last = d->is_last;
return htp_gzip_decompressor_decompress((htp_decompressor_gzip_t *)drec->super.next, &d3);

callback_rc = htp_gzip_decompressor_decompress((htp_decompressor_gzip_t *)drec->super.next, &d2);
} else {
// Send decompressed data to the callback.
htp_status_t callback_rc = drec->super.callback(&d2);
if (callback_rc != HTP_OK) {
inflateEnd(&drec->stream);
drec->zlib_initialized = 0;

return callback_rc;
}
callback_rc = drec->super.callback(&d2);
}
if (callback_rc != HTP_OK) {
htp_gzip_decompressor_end(drec);
return callback_rc;
}
drec->stream.avail_out = GZIP_BUF_SIZE;
drec->stream.next_out = drec->buffer;
// TODO Handle trailer.

return HTP_OK;
}
else if (rc != Z_OK) {
htp_log(d->tx->connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "GZip decompressor: inflate failed with %d", rc);
inflateEnd(&drec->stream);
if (drec->zlib_initialized == HTP_COMPRESSION_LZMA) {
LzmaDec_Free(&drec->state, &lzma_Alloc);
// so as to clean zlib ressources after restart
drec->zlib_initialized = HTP_COMPRESSION_NONE;
} else {
inflateEnd(&drec->stream);
}

// see if we want to restart the decompressor
if (htp_gzip_decompressor_restart(drec,
Expand All @@ -311,11 +382,14 @@ static htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_gzip_t *dr
d2.len = d->len;
d2.is_last = d->is_last;

htp_status_t callback_rc = drec->super.callback(&d2);
callback_rc = drec->super.callback(&d2);
if (callback_rc != HTP_OK) {
return HTP_ERROR;
}

drec->stream.avail_out = GZIP_BUF_SIZE;
drec->stream.next_out = drec->buffer;

/* successfully passed through, lets continue doing that */
drec->passthrough = 1;
return HTP_OK;
Expand All @@ -333,10 +407,7 @@ static htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_gzip_t *dr
static void htp_gzip_decompressor_destroy(htp_decompressor_gzip_t *drec) {
if (drec == NULL) return;

if (drec->zlib_initialized) {
inflateEnd(&drec->stream);
drec->zlib_initialized = 0;
}
htp_gzip_decompressor_end(drec);

free(drec->buffer);
free(drec);
Expand Down Expand Up @@ -366,19 +437,36 @@ htp_decompressor_t *htp_gzip_decompressor_create(htp_connp_t *connp, enum htp_co
// Initialize zlib.
int rc;

if (format == HTP_COMPRESSION_DEFLATE) {
// Negative values activate raw processing,
// which is what we need for deflate.
rc = inflateInit2(&drec->stream, -15);
} else {
// Increased windows size activates gzip header processing.
rc = inflateInit2(&drec->stream, 15 + 32);
switch (format) {
case HTP_COMPRESSION_LZMA:
if (connp->cfg->lzma_memlimit > 0) {
LzmaDec_Construct(&drec->state);
} else {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "LZMA decompression disabled");
drec->passthrough = 1;
}
rc = Z_OK;
break;
case HTP_COMPRESSION_DEFLATE:
// Negative values activate raw processing,
// which is what we need for deflate.
rc = inflateInit2(&drec->stream, -15);
break;
case HTP_COMPRESSION_GZIP:
// Increased windows size activates gzip header processing.
rc = inflateInit2(&drec->stream, 15 + 32);
break;
default:
// do nothing
rc = Z_DATA_ERROR;
}

if (rc != Z_OK) {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "GZip decompressor: inflateInit2 failed with code %d", rc);

inflateEnd(&drec->stream);
if (format == HTP_COMPRESSION_DEFLATE || format == HTP_COMPRESSION_GZIP) {
inflateEnd(&drec->stream);
}
free(drec->buffer);
free(drec);

Expand Down
6 changes: 4 additions & 2 deletions htp/htp_decompressors.h
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ extern "C" {
#endif

#include <zlib.h>
#include "lzma/LzmaDec.h"

typedef struct htp_decompressor_gzip_t htp_decompressor_gzip_t;
typedef struct htp_decompressor_t htp_decompressor_t;
Expand All @@ -66,11 +67,12 @@ struct htp_decompressor_gzip_t {
int initialized;
#endif
int zlib_initialized;
uint8_t header[10];
uint8_t header_len;
uint8_t restart; /**< deflate restarted to try rfc1950 instead of 1951 */
uint8_t passthrough; /**< decompression failed, pass through raw data */
z_stream stream;
uint8_t header[LZMA_PROPS_SIZE + 8];
uint8_t header_len;
CLzmaDec state;
unsigned char *buffer;
unsigned long crc;
};
Expand Down
32 changes: 23 additions & 9 deletions htp/htp_list.c
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,22 @@

// Array-backed list

htp_status_t htp_list_array_init(htp_list_t *l, size_t size) {
// Allocate the initial batch of elements.
l->elements = malloc(size * sizeof (void *));
if (l->elements == NULL) {
return HTP_ERROR;
}

// Initialize the structure.
l->first = 0;
l->last = 0;
l->current_size = 0;
l->max_size = size;

return HTP_OK;
}

htp_list_t *htp_list_array_create(size_t size) {
// It makes no sense to create a zero-size list.
if (size == 0) return NULL;
Expand All @@ -50,19 +66,11 @@ htp_list_t *htp_list_array_create(size_t size) {
htp_list_array_t *l = calloc(1, sizeof (htp_list_array_t));
if (l == NULL) return NULL;

// Allocate the initial batch of elements.
l->elements = malloc(size * sizeof (void *));
if (l->elements == NULL) {
if (htp_list_array_init(l, size) == HTP_ERROR) {
free(l);
return NULL;
}

// Initialize the structure.
l->first = 0;
l->last = 0;
l->current_size = 0;
l->max_size = size;

return (htp_list_t *) l;
}

Expand All @@ -82,6 +90,12 @@ void htp_list_array_destroy(htp_list_array_t *l) {
free(l);
}

void htp_list_array_release(htp_list_array_t *l) {
if (l == NULL) return;

free(l->elements);
}

void *htp_list_array_get(const htp_list_array_t *l, size_t idx) {
if (l == NULL) return NULL;
if (idx + 1 > l->current_size) return NULL;
Expand Down
20 changes: 20 additions & 0 deletions htp/htp_list.h
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,10 @@ extern "C" {
#define htp_list_t htp_list_array_t
#define htp_list_add htp_list_array_push
#define htp_list_create htp_list_array_create
#define htp_list_init htp_list_array_init
#define htp_list_clear htp_list_array_clear
#define htp_list_destroy htp_list_array_destroy
#define htp_list_release htp_list_array_release
#define htp_list_get htp_list_array_get
#define htp_list_pop htp_list_array_pop
#define htp_list_push htp_list_array_push
Expand All @@ -75,6 +77,15 @@ typedef struct htp_list_linked_t htp_list_linked_t;
*/
htp_list_array_t *htp_list_array_create(size_t size);

/**
* Initialize an array-backed list.
*
* @param[in] l
* @param[in] size
* @return HTP_OK or HTP_ERROR if allocation failed
*/
htp_status_t htp_list_array_init(htp_list_array_t *l, size_t size);

/**
* Remove all elements from the list. It is the responsibility of the caller
* to iterate over list elements and deallocate them if necessary, prior to
Expand All @@ -92,6 +103,15 @@ void htp_list_array_clear(htp_list_array_t *l);
*/
void htp_list_array_destroy(htp_list_array_t *l);

/**
* Free the memory occupied by this list, except itself.
* This function assumes the elements held by the list
* were freed beforehand.
*
* @param[in] l
*/
void htp_list_array_release(htp_list_array_t *l);

/**
* Find the element at the given index.
*
Expand Down
10 changes: 9 additions & 1 deletion htp/htp_private.h
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@ extern "C" {
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdint.h>

#include "htp_config_auto_gen.h"
#include "htp.h"
#include "htp_config_private.h"
#include "htp_connection_parser_private.h"
Expand All @@ -76,6 +78,12 @@ extern "C" {
#define LF '\n'
#endif

// 1048576 is 1 Mbyte
#define HTP_LZMA_MEMLIMIT 1048576
//deflate max ratio is about 1000
#define HTP_COMPRESSION_BOMB_RATIO 2048
#define HTP_COMPRESSION_BOMB_LIMIT 1048576

#define HTP_FIELD_LIMIT_HARD 18000
#define HTP_FIELD_LIMIT_SOFT 9000

Expand Down Expand Up @@ -170,7 +178,7 @@ void htp_normalize_uri_path_inplace(bstr *s);
void htp_utf8_decode_path_inplace(htp_cfg_t *cfg, htp_tx_t *tx, bstr *path);
void htp_utf8_validate_path(htp_tx_t *tx, bstr *path);

int64_t htp_parse_content_length(bstr *b);
int64_t htp_parse_content_length(bstr *b, htp_connp_t *connp);
int64_t htp_parse_chunked_length(unsigned char *data, size_t len);
int64_t htp_parse_positive_integer_whitespace(unsigned char *data, size_t len, int base);
int htp_parse_status(bstr *status);
Expand Down
74 changes: 72 additions & 2 deletions htp/htp_request.c
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,6 @@ htp_status_t htp_connp_REQ_CONNECT_PROBE_DATA(htp_connp_t *connp) {
unsigned char *data;
size_t len;
if (htp_connp_req_consolidate_data(connp, &data, &len) != HTP_OK) {
fprintf(stderr, "htp_connp_req_consolidate_data fail");
return HTP_ERROR;
}
#ifdef HTP_DEBUG
Expand Down Expand Up @@ -358,7 +357,7 @@ htp_status_t htp_connp_REQ_CONNECT_PROBE_DATA(htp_connp_t *connp) {
#ifdef HTP_DEBUG
fprint_raw_data(stderr, "htp_connp_REQ_CONNECT_PROBE_DATA: tunnel contains plain text HTTP", data, len);
#endif
connp->in_state = htp_connp_REQ_FINALIZE;
return htp_tx_state_request_complete(connp->in_tx);
} else {
#ifdef HTP_DEBUG
fprint_raw_data(stderr, "htp_connp_REQ_CONNECT_PROBE_DATA: tunnel is not HTTP", data, len);
Expand Down Expand Up @@ -615,6 +614,23 @@ htp_status_t htp_connp_REQ_BODY_DETERMINE(htp_connp_t *connp) {
*/
htp_status_t htp_connp_REQ_HEADERS(htp_connp_t *connp) {
for (;;) {
if (connp->in_status == HTP_STREAM_CLOSED) {
// Parse previous header, if any.
if (connp->in_header != NULL) {
if (connp->cfg->process_request_header(connp, bstr_ptr(connp->in_header),
bstr_len(connp->in_header)) != HTP_OK)
return HTP_ERROR;
bstr_free(connp->in_header);
connp->in_header = NULL;
}

htp_connp_req_clear_buffer(connp);

connp->in_tx->request_progress = HTP_REQUEST_TRAILER;

// We've seen all the request headers.
return htp_tx_state_request_headers(connp->in_tx);
}
IN_COPY_BYTE_OR_RETURN(connp);

// Have we reached the end of the line?
Expand Down Expand Up @@ -715,6 +731,26 @@ htp_status_t htp_connp_REQ_PROTOCOL(htp_connp_t *connp) {
connp->in_state = htp_connp_REQ_HEADERS;
connp->in_tx->request_progress = HTP_REQUEST_HEADERS;
} else {
// Let's check if the protocol was simply missing
int64_t pos = connp->in_current_read_offset;
int afterspaces = 0;
// Probe if data looks like a header line
while (pos < connp->in_current_len) {
if (connp->in_current_data[pos] == ':') {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request line: missing protocol");
connp->in_tx->is_protocol_0_9 = 0;
// Switch to request header parsing.
connp->in_state = htp_connp_REQ_HEADERS;
connp->in_tx->request_progress = HTP_REQUEST_HEADERS;
return HTP_OK;
} else if (htp_is_lws(connp->in_current_data[pos])) {
// Allows spaces after header name
afterspaces = 1;
} else if (htp_is_space(connp->in_current_data[pos]) || afterspaces == 1) {
break;
}
pos++;
}
// We're done with this request.
connp->in_state = htp_connp_REQ_FINALIZE;
}
Expand Down Expand Up @@ -792,6 +828,40 @@ htp_status_t htp_connp_REQ_LINE(htp_connp_t *connp) {
}

htp_status_t htp_connp_REQ_FINALIZE(htp_connp_t *connp) {
size_t bytes_left = connp->in_current_len - connp->in_current_read_offset;

if (bytes_left > 0) {
// If we have more bytes
// Either it is request pipelining
// Or we interpret it as body data
int64_t pos = connp->in_current_read_offset;
int64_t mstart = 0;
// skip past leading whitespace. IIS allows this
while ((pos < connp->in_current_len) && htp_is_space(connp->in_current_data[pos]))
pos++;
if (pos < connp->in_current_len) {
mstart = pos;
// The request method starts at the beginning of the
// line and ends with the first whitespace character.
while ((pos < connp->in_current_len) && (!htp_is_space(connp->in_current_data[pos])))
pos++;

int methodi = HTP_M_UNKNOWN;
bstr *method = bstr_dup_mem(connp->in_current_data + mstart, pos - mstart);
if (method) {
methodi = htp_convert_method_to_number(method);
bstr_free(method);
}
if (methodi == HTP_M_UNKNOWN) {
// Interpret remaining bytes as body data
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Unexpected request body");
connp->in_tx->request_progress = HTP_REQUEST_BODY;
connp->in_state = htp_connp_REQ_BODY_IDENTITY;
connp->in_body_data_left = bytes_left;
return HTP_OK;
}
}
}
return htp_tx_state_request_complete(connp->in_tx);
}

Expand Down
68 changes: 54 additions & 14 deletions htp/htp_request_generic.c
Original file line number Diff line number Diff line change
Expand Up @@ -70,27 +70,55 @@ htp_status_t htp_process_request_header_generic(htp_connp_t *connp, unsigned cha
if (h_existing != NULL) {
// TODO Do we want to have a list of the headers that are
// allowed to be combined in this way?

// Add to the existing header.
bstr *new_value = bstr_expand(h_existing->value, bstr_len(h_existing->value) + 2 + bstr_len(h->value));
if (new_value == NULL) {
bstr_free(h->name);
bstr_free(h->value);
free(h);
return HTP_ERROR;
if ((h_existing->flags & HTP_FIELD_REPEATED) == 0) {
// This is the second occurence for this header.
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Repetition for header");
} else {
// For simplicity reasons, we count the repetitions of all headers
if (connp->in_tx->req_header_repetitions < HTP_MAX_HEADERS_REPETITIONS) {
connp->in_tx->req_header_repetitions++;
} else {
bstr_free(h->name);
bstr_free(h->value);
free(h);
return HTP_OK;
}
}
// Keep track of repeated same-name headers.
h_existing->flags |= HTP_FIELD_REPEATED;

h_existing->value = new_value;
bstr_add_mem_noex(h_existing->value, ", ", 2);
bstr_add_noex(h_existing->value, h->value);
// Having multiple C-L headers is against the RFC but
// servers may ignore the subsequent headers if the values are the same.
if (bstr_cmp_c_nocase(h->name, "Content-Length") == 0) {
// Don't use string comparison here because we want to
// ignore small formatting differences.

int64_t existing_cl = htp_parse_content_length(h_existing->value, NULL);
int64_t new_cl = htp_parse_content_length(h->value, NULL);
// Ambiguous response C-L value.
if ((existing_cl == -1) || (new_cl == -1) || (existing_cl != new_cl)) {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Ambiguous request C-L value");
}
// Ignoring the new C-L header that has the same value as the previous ones.
} else {
// Add to the existing header.
bstr *new_value = bstr_expand(h_existing->value, bstr_len(h_existing->value) + 2 + bstr_len(h->value));
if (new_value == NULL) {
bstr_free(h->name);
bstr_free(h->value);
free(h);
return HTP_ERROR;
}

h_existing->value = new_value;
bstr_add_mem_noex(h_existing->value, ", ", 2);
bstr_add_noex(h_existing->value, h->value);
}

// The new header structure is no longer needed.
bstr_free(h->name);
bstr_free(h->value);
free(h);

// Keep track of repeated same-name headers.
h_existing->flags |= HTP_FIELD_REPEATED;
} else {
// Add as a new header.
if (htp_table_add(connp->in_tx->request_headers, h->name, h) != HTP_OK) {
Expand Down Expand Up @@ -315,16 +343,21 @@ htp_status_t htp_parse_request_line_generic_ex(htp_connp_t *connp, int nul_termi
}
pos++;
}
// Too much performance overhead for fuzzing
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
if (bad_delim) {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request line: non-compliant delimiter between Method and URI");
}
#endif

// Is there anything after the request method?
if (pos == len) {
// No, this looks like a HTTP/0.9 request.

tx->is_protocol_0_9 = 1;
tx->request_protocol_number = HTP_PROTOCOL_0_9;
if (tx->request_method_number == HTP_M_UNKNOWN)
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request line: unknown method only");

return HTP_OK;
}
Expand All @@ -347,10 +380,13 @@ htp_status_t htp_parse_request_line_generic_ex(htp_connp_t *connp, int nul_termi
pos = start;
while ((pos < len) && (!htp_is_space(data[pos]))) pos++;
}
// Too much performance overhead for fuzzing
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
if (bad_delim) {
// warn regardless if we've seen non-compliant chars
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request line: URI contains non-compliant delimiter");
}
#endif

tx->request_uri = bstr_dup_mem(data + start, pos - start);
if (tx->request_uri == NULL) return HTP_ERROR;
Expand All @@ -368,6 +404,8 @@ htp_status_t htp_parse_request_line_generic_ex(htp_connp_t *connp, int nul_termi

tx->is_protocol_0_9 = 1;
tx->request_protocol_number = HTP_PROTOCOL_0_9;
if (tx->request_method_number == HTP_M_UNKNOWN)
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request line: unknown method and no protocol");

return HTP_OK;
}
Expand All @@ -377,6 +415,8 @@ htp_status_t htp_parse_request_line_generic_ex(htp_connp_t *connp, int nul_termi
if (tx->request_protocol == NULL) return HTP_ERROR;

tx->request_protocol_number = htp_parse_protocol(tx->request_protocol);
if (tx->request_method_number == HTP_M_UNKNOWN && tx->request_protocol_number == HTP_PROTOCOL_INVALID)
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request line: unknown method and invalid protocol");

#ifdef HTP_DEBUG
fprint_raw_data(stderr, __func__, bstr_ptr(tx->request_protocol), bstr_len(tx->request_protocol));
Expand Down
201 changes: 157 additions & 44 deletions htp/htp_response.c

Large diffs are not rendered by default.

38 changes: 26 additions & 12 deletions htp/htp_response_generic.c
Original file line number Diff line number Diff line change
Expand Up @@ -171,9 +171,9 @@ htp_status_t htp_parse_response_header_generic(htp_connp_t *connp, htp_header_t

name_end = colon_pos;

// Ignore LWS after field-name.
// Ignore unprintable after field-name.
prev = name_end;
while ((prev > name_start) && (htp_is_lws(data[prev - 1]))) {
while ((prev > name_start) && (data[prev - 1] <= 0x20)) {
prev--;
name_end--;

Expand Down Expand Up @@ -215,6 +215,12 @@ htp_status_t htp_parse_response_header_generic(htp_connp_t *connp, htp_header_t

i++;
}
for (i = value_start; i < value_end; i++) {
if (data[i] == 0) {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Response header value contains null.");
break;
}
}

// Now extract the name and the value.
h->name = bstr_dup_mem(data + name_start, name_end - name_start);
Expand Down Expand Up @@ -256,8 +262,22 @@ htp_status_t htp_process_response_header_generic(htp_connp_t *connp, unsigned ch
htp_header_t *h_existing = htp_table_get(connp->out_tx->response_headers, h->name);
if (h_existing != NULL) {
// Keep track of repeated same-name headers.
if ((h_existing->flags & HTP_FIELD_REPEATED) == 0) {
// This is the second occurence for this header.
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Repetition for header");
} else {
// For simplicity reasons, we count the repetitions of all headers
if (connp->out_tx->res_header_repetitions < HTP_MAX_HEADERS_REPETITIONS) {
connp->out_tx->res_header_repetitions++;
} else {
bstr_free(h->name);
bstr_free(h->value);
free(h);
return HTP_OK;
}
}
h_existing->flags |= HTP_FIELD_REPEATED;

// Having multiple C-L headers is against the RFC but many
// browsers ignore the subsequent headers if the values are the same.
if (bstr_cmp_c_nocase(h->name, "Content-Length") == 0) {
Expand All @@ -266,17 +286,11 @@ htp_status_t htp_process_response_header_generic(htp_connp_t *connp, unsigned ch

int64_t existing_cl, new_cl;

existing_cl = htp_parse_content_length(h_existing->value);
new_cl = htp_parse_content_length(h->value);
existing_cl = htp_parse_content_length(h_existing->value, NULL);
new_cl = htp_parse_content_length(h->value, NULL);
if ((existing_cl == -1) || (new_cl == -1) || (existing_cl != new_cl)) {
// Ambiguous response C-L value.
htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Ambiguous response C-L value");

bstr_free(h->name);
bstr_free(h->value);
free(h);

return HTP_ERROR;
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Ambiguous response C-L value");
}

// Ignoring the new C-L header that has the same value as the previous ones.
Expand Down
48 changes: 23 additions & 25 deletions htp/htp_table.c
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@

static htp_status_t _htp_table_add(htp_table_t *table, const bstr *key, const void *element) {
// Add key.
if (htp_list_add(table->list, (void *)key) != HTP_OK) return HTP_ERROR;
if (htp_list_add(&table->list, (void *)key) != HTP_OK) return HTP_ERROR;

// Add element.
if (htp_list_add(table->list, (void *)element) != HTP_OK) {
htp_list_pop(table->list);
if (htp_list_add(&table->list, (void *)element) != HTP_OK) {
htp_list_pop(&table->list);
return HTP_ERROR;
}

Expand Down Expand Up @@ -130,21 +130,21 @@ void htp_table_clear(htp_table_t *table) {
// Free the table keys, but only if we're managing them.
if ((table->alloc_type == HTP_TABLE_KEYS_COPIED)||(table->alloc_type == HTP_TABLE_KEYS_ADOPTED)) {
bstr *key = NULL;
for (size_t i = 0, n = htp_list_size(table->list); i < n; i += 2) {
key = htp_list_get(table->list, i);
for (size_t i = 0, n = htp_list_size(&table->list); i < n; i += 2) {
key = htp_list_get(&table->list, i);
bstr_free(key);
}
}

htp_list_clear(table->list);
htp_list_clear(&table->list);
}

void htp_table_clear_ex(htp_table_t *table) {
if (table == NULL) return;

// This function does not free table keys.

htp_list_clear(table->list);
htp_list_clear(&table->list);
}

htp_table_t *htp_table_create(size_t size) {
Expand All @@ -156,8 +156,7 @@ htp_table_t *htp_table_create(size_t size) {
table->alloc_type = HTP_TABLE_KEYS_ALLOC_UKNOWN;

// Use a list behind the scenes.
table->list = htp_list_array_create(size * 2);
if (table->list == NULL) {
if (htp_list_init(&table->list, size * 2) == HTP_ERROR) {
free(table);
return NULL;
}
Expand All @@ -170,8 +169,7 @@ void htp_table_destroy(htp_table_t *table) {

htp_table_clear(table);

htp_list_destroy(table->list);
table->list = NULL;
htp_list_array_release(&table->list);

free(table);
}
Expand All @@ -191,9 +189,9 @@ void *htp_table_get(const htp_table_t *table, const bstr *key) {

// Iterate through the list, comparing
// keys with the parameter, return data if found.
for (size_t i = 0, n = htp_list_size(table->list); i < n; i += 2) {
bstr *key_candidate = htp_list_get(table->list, i);
void *element = htp_list_get(table->list, i + 1);
for (size_t i = 0, n = htp_list_size(&table->list); i < n; i += 2) {
bstr *key_candidate = htp_list_get(&table->list, i);
void *element = htp_list_get(&table->list, i + 1);
if (bstr_cmp_nocase(key_candidate, key) == 0) {
return element;
}
Expand All @@ -207,10 +205,10 @@ void *htp_table_get_c(const htp_table_t *table, const char *ckey) {

// Iterate through the list, comparing
// keys with the parameter, return data if found.
for (size_t i = 0, n = htp_list_size(table->list); i < n; i += 2) {
bstr *key_candidate = htp_list_get(table->list, i);
void *element = htp_list_get(table->list, i + 1);
if (bstr_cmp_c_nocase(key_candidate, ckey) == 0) {
for (size_t i = 0, n = htp_list_size(&table->list); i < n; i += 2) {
bstr *key_candidate = htp_list_get(&table->list, i);
void *element = htp_list_get(&table->list, i + 1);
if (bstr_cmp_c_nocasenorzero(key_candidate, ckey) == 0) {
return element;
}
}
Expand All @@ -221,23 +219,23 @@ void *htp_table_get_c(const htp_table_t *table, const char *ckey) {
void *htp_table_get_index(const htp_table_t *table, size_t idx, bstr **key) {
if (table == NULL) return NULL;

if (idx >= htp_list_size(table->list)) return NULL;
if (idx >= htp_list_size(&table->list)) return NULL;

if (key != NULL) {
*key = htp_list_get(table->list, idx * 2);
*key = htp_list_get(&table->list, idx * 2);
}

return htp_list_get(table->list, (idx * 2) + 1);
return htp_list_get(&table->list, (idx * 2) + 1);
}

void *htp_table_get_mem(const htp_table_t *table, const void *key, size_t key_len) {
if ((table == NULL)||(key == NULL)) return NULL;

// Iterate through the list, comparing
// keys with the parameter, return data if found.
for (size_t i = 0, n = htp_list_size(table->list); i < n; i += 2) {
bstr *key_candidate = htp_list_get(table->list, i);
void *element = htp_list_get(table->list, i + 1);
for (size_t i = 0, n = htp_list_size(&table->list); i < n; i += 2) {
bstr *key_candidate = htp_list_get(&table->list, i);
void *element = htp_list_get(&table->list, i + 1);
if (bstr_cmp_mem_nocase(key_candidate, key, key_len) == 0) {
return element;
}
Expand All @@ -248,5 +246,5 @@ void *htp_table_get_mem(const htp_table_t *table, const void *key, size_t key_le

size_t htp_table_size(const htp_table_t *table) {
if (table == NULL) return 0;
return htp_list_size(table->list) / 2;
return htp_list_size(&table->list) / 2;
}
2 changes: 1 addition & 1 deletion htp/htp_table_private.h
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ enum htp_table_alloc_t {

struct htp_table_t {
/** Table key and value pairs are stored in this list; name first, then value. */
htp_list_t *list;
htp_list_t list;

/**
* Key management strategy. Initially set to HTP_TABLE_KEYS_ALLOC_UKNOWN. The
Expand Down
23 changes: 19 additions & 4 deletions htp/htp_transaction.c
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ static htp_status_t htp_tx_process_request_headers(htp_tx_t *tx) {
}

// Get the body length.
tx->request_content_length = htp_parse_content_length(cl->value);
tx->request_content_length = htp_parse_content_length(cl->value, tx->connp);
if (tx->request_content_length < 0) {
tx->request_transfer_coding = HTP_CODING_INVALID;
tx->flags |= HTP_REQUEST_INVALID_C_L;
Expand Down Expand Up @@ -789,6 +789,13 @@ static htp_status_t htp_tx_res_process_body_data_decompressor_callback(htp_tx_da
// Invoke all callbacks.
htp_status_t rc = htp_res_run_hook_body_data(d->tx->connp, d);
if (rc != HTP_OK) return HTP_ERROR;
if (d->tx->response_entity_len > d->tx->connp->cfg->compression_bomb_limit &&
d->tx->response_entity_len > HTP_COMPRESSION_BOMB_RATIO * d->tx->response_message_len) {
htp_log(d->tx->connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
"Compression bomb: decompressed %"PRId64" bytes out of %"PRId64,
d->tx->response_entity_len, d->tx->response_message_len);
return HTP_ERROR;
}

return HTP_OK;
}
Expand Down Expand Up @@ -822,6 +829,7 @@ htp_status_t htp_tx_res_process_body_data_ex(htp_tx_t *tx, const void *data, siz
switch (tx->response_content_encoding_processing) {
case HTP_COMPRESSION_GZIP:
case HTP_COMPRESSION_DEFLATE:
case HTP_COMPRESSION_LZMA:
// In severe memory stress these could be NULL
if (tx->connp->out_decompressor == NULL || tx->connp->out_decompressor->decompress == NULL)
return HTP_ERROR;
Expand Down Expand Up @@ -1166,11 +1174,15 @@ htp_status_t htp_tx_state_response_headers(htp_tx_t *tx) {
htp_header_t *ce = htp_table_get_c(tx->response_headers, "content-encoding");
if (ce != NULL) {
/* fast paths: regular gzip and friends */
if ((bstr_cmp_c_nocase(ce->value, "gzip") == 0) || (bstr_cmp_c_nocase(ce->value, "x-gzip") == 0)) {
if ((bstr_cmp_c_nocasenorzero(ce->value, "gzip") == 0) ||
(bstr_cmp_c_nocasenorzero(ce->value, "x-gzip") == 0)) {
tx->response_content_encoding = HTP_COMPRESSION_GZIP;
} else if ((bstr_cmp_c_nocase(ce->value, "deflate") == 0) || (bstr_cmp_c_nocase(ce->value, "x-deflate") == 0)) {
} else if ((bstr_cmp_c_nocasenorzero(ce->value, "deflate") == 0) ||
(bstr_cmp_c_nocasenorzero(ce->value, "x-deflate") == 0)) {
tx->response_content_encoding = HTP_COMPRESSION_DEFLATE;
} else if (bstr_cmp_c_nocase(ce->value, "inflate") == 0) {
} else if (bstr_cmp_c_nocasenorzero(ce->value, "lzma") == 0) {
tx->response_content_encoding = HTP_COMPRESSION_LZMA;
} else if (bstr_cmp_c_nocasenorzero(ce->value, "inflate") == 0) {
// ignore
} else {
/* exceptional cases: enter slow path */
Expand Down Expand Up @@ -1207,6 +1219,7 @@ htp_status_t htp_tx_state_response_headers(htp_tx_t *tx) {
// supported algorithms.
if ((tx->response_content_encoding_processing == HTP_COMPRESSION_GZIP) ||
(tx->response_content_encoding_processing == HTP_COMPRESSION_DEFLATE) ||
(tx->response_content_encoding_processing == HTP_COMPRESSION_LZMA) ||
ce_multi_comp)
{
if (tx->connp->out_decompressor != NULL) {
Expand Down Expand Up @@ -1268,6 +1281,8 @@ htp_status_t htp_tx_state_response_headers(htp_tx_t *tx) {
"C-E deflate has abnormal value");
}
cetype = HTP_COMPRESSION_DEFLATE;
} else if (bstr_util_cmp_mem(tok, tok_len, "lzma", 4) == 0) {
cetype = HTP_COMPRESSION_LZMA;
} else if (bstr_util_cmp_mem(tok, tok_len, "inflate", 7) == 0) {
cetype = HTP_COMPRESSION_NONE;
} else {
Expand Down
61 changes: 49 additions & 12 deletions htp/htp_util.c
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ int htp_is_token(int c) {
}

/**
* Remove all line terminators (LF or CRLF) from
* Remove all line terminators (LF, CR or CRLF) from
* the end of the line provided as input.
*
* @return 0 if nothing was removed, 1 if one or more LF characters were removed, or
Expand All @@ -139,6 +139,9 @@ int htp_chomp(unsigned char *data, size_t *len) {
(*len)--;
r = 2;
}
} else if (data[*len - 1] == CR) {
(*len)--;
r = 1;
} else return r;
}

Expand Down Expand Up @@ -209,15 +212,15 @@ int htp_convert_method_to_number(bstr *method) {
}

/**
* Is the given line empty? This function expects the line to have
* a terminating LF.
* Is the given line empty?
*
* @param[in] data
* @param[in] len
* @return 0 or 1
*/
int htp_is_line_empty(unsigned char *data, size_t len) {
if ((len == 1) || ((len == 2) && (data[0] == CR))) {
if ((len == 1) ||
((len == 2) && (data[0] == CR) && (data[1] == LF))) {
return 1;
}

Expand Down Expand Up @@ -250,8 +253,32 @@ int htp_is_line_whitespace(unsigned char *data, size_t len) {
* @param[in] b
* @return Content-Length as a number, or -1 on error.
*/
int64_t htp_parse_content_length(bstr *b) {
return htp_parse_positive_integer_whitespace((unsigned char *) bstr_ptr(b), bstr_len(b), 10);
int64_t htp_parse_content_length(bstr *b, htp_connp_t *connp) {
size_t len = bstr_len(b);
unsigned char * data = (unsigned char *) bstr_ptr(b);
size_t pos = 0;
int64_t r = 0;

if (len == 0) return -1003;

// Ignore junk before
while ((pos < len) && (data[pos] < '0' || data[pos] > '9')) {
if (!htp_is_lws(data[pos]) && connp != NULL && r == 0) {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0,
"C-L value with extra data in the beginnning");
r = -1;
}
pos++;
}
if (pos == len) return -1001;

r = bstr_util_mem_to_pint(data + pos, len - pos, 10, &pos);
// Ok to have junk afterwards
if (pos < len && connp != NULL) {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0,
"C-L value with extra data in the end");
}
return r;
}

/**
Expand Down Expand Up @@ -425,7 +452,7 @@ int htp_connp_is_line_folded(unsigned char *data, size_t len) {
}

int htp_is_folding_char(int c) {
if (htp_is_lws(c)) return 1;
if (htp_is_lws(c) || c == 0) return 1;
else return 0;
}

Expand All @@ -452,6 +479,13 @@ int htp_connp_is_line_terminator(htp_connp_t *connp, unsigned char *data, size_t
if (htp_is_line_empty(data, len)) {
return 1;
}
// Only space is terminator if terminator does not follow right away
if (len == 2 && htp_is_lws(data[0]) && data[1] == LF) {
if (connp->out_current_read_offset < connp->out_current_len &&
connp->out_current_data[connp->out_current_read_offset] != LF) {
return 1;
}
}
break;
}

Expand Down Expand Up @@ -2366,14 +2400,17 @@ int htp_treat_response_line_as_body(const uint8_t *data, size_t len) {
// Firefox 3.5.x: (?i)^\s*http
// IE: (?i)^\s*http\s*/
// Safari: ^HTTP/\d+\.\d+\s+\d{3}
size_t pos = 0;

if (data == NULL) return 1;
if (len < 4) return 1;
while ((pos < len) && (htp_is_space(data[pos]) || data[pos] == 0)) pos++;

if (len < pos + 4) return 1;

if ((data[0] != 'H') && (data[0] != 'h')) return 1;
if ((data[1] != 'T') && (data[1] != 't')) return 1;
if ((data[2] != 'T') && (data[2] != 't')) return 1;
if ((data[3] != 'P') && (data[3] != 'p')) return 1;
if ((data[pos] != 'H') && (data[pos] != 'h')) return 1;
if ((data[pos+1] != 'T') && (data[pos+1] != 't')) return 1;
if ((data[pos+2] != 'T') && (data[pos+2] != 't')) return 1;
if ((data[pos+3] != 'P') && (data[pos+3] != 'p')) return 1;

return 0;
}
Expand Down
375 changes: 375 additions & 0 deletions htp/lzma/7zTypes.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,375 @@
/* 7zTypes.h -- Basic types
2018-08-04 : Igor Pavlov : Public domain */

#ifndef __7Z_TYPES_H
#define __7Z_TYPES_H

#ifdef _WIN32
/* #include <windows.h> */
#endif

#include <stddef.h>
#include <zconf.h>

#ifndef EXTERN_C_BEGIN
#ifdef __cplusplus
#define EXTERN_C_BEGIN extern "C" {
#define EXTERN_C_END }
#else
#define EXTERN_C_BEGIN
#define EXTERN_C_END
#endif
#endif

EXTERN_C_BEGIN

#define SZ_OK 0

#define SZ_ERROR_DATA 1
#define SZ_ERROR_MEM 2
#define SZ_ERROR_CRC 3
#define SZ_ERROR_UNSUPPORTED 4
#define SZ_ERROR_PARAM 5
#define SZ_ERROR_INPUT_EOF 6
#define SZ_ERROR_OUTPUT_EOF 7
#define SZ_ERROR_READ 8
#define SZ_ERROR_WRITE 9
#define SZ_ERROR_PROGRESS 10
#define SZ_ERROR_FAIL 11
#define SZ_ERROR_THREAD 12

#define SZ_ERROR_ARCHIVE 16
#define SZ_ERROR_NO_ARCHIVE 17

typedef int SRes;


#ifdef _WIN32

/* typedef DWORD WRes; */
typedef unsigned WRes;
#define MY_SRes_HRESULT_FROM_WRes(x) HRESULT_FROM_WIN32(x)

#else

typedef int WRes;
#define MY__FACILITY_WIN32 7
#define MY__FACILITY__WRes MY__FACILITY_WIN32
#define MY_SRes_HRESULT_FROM_WRes(x) ((HRESULT)(x) <= 0 ? ((HRESULT)(x)) : ((HRESULT) (((x) & 0x0000FFFF) | (MY__FACILITY__WRes << 16) | 0x80000000)))

#endif


#ifndef RINOK
#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; }
#endif

typedef short Int16;
typedef unsigned short UInt16;

#ifdef _LZMA_UINT32_IS_ULONG
typedef long Int32;
typedef unsigned long UInt32;
#else
typedef int Int32;
typedef unsigned int UInt32;
#endif

#ifdef _SZ_NO_INT_64

/* define _SZ_NO_INT_64, if your compiler doesn't support 64-bit integers.
NOTES: Some code will work incorrectly in that case! */

typedef long Int64;
typedef unsigned long UInt64;

#else

#if defined(_MSC_VER) || defined(__BORLANDC__)
typedef __int64 Int64;
typedef unsigned __int64 UInt64;
#define UINT64_CONST(n) n
#else
typedef long long int Int64;
typedef unsigned long long int UInt64;
#define UINT64_CONST(n) n ## ULL
#endif

#endif

#ifdef _LZMA_NO_SYSTEM_SIZE_T
typedef UInt32 SizeT;
#else
typedef size_t SizeT;
#endif

typedef int BoolInt;
/* typedef BoolInt Bool; */
#define True 1
#define False 0


#ifdef _WIN32
#define MY_STD_CALL __stdcall
#else
#define MY_STD_CALL
#endif

#ifdef _MSC_VER

#if _MSC_VER >= 1300
#define MY_NO_INLINE __declspec(noinline)
#else
#define MY_NO_INLINE
#endif

#define MY_FORCE_INLINE __forceinline

#define MY_CDECL __cdecl
#define MY_FAST_CALL __fastcall

#else

#define MY_NO_INLINE
#define MY_FORCE_INLINE
#define MY_CDECL
#define MY_FAST_CALL

/* inline keyword : for C++ / C99 */

/* GCC, clang: */
/*
#if defined (__GNUC__) && (__GNUC__ >= 4)
#define MY_FORCE_INLINE __attribute__((always_inline))
#define MY_NO_INLINE __attribute__((noinline))
#endif
*/

#endif


/* The following interfaces use first parameter as pointer to structure */

typedef struct IByteIn IByteIn;
struct IByteIn
{
Byte (*Read)(const IByteIn *p); /* reads one byte, returns 0 in case of EOF or error */
};
#define IByteIn_Read(p) (p)->Read(p)


typedef struct IByteOut IByteOut;
struct IByteOut
{
void (*Write)(const IByteOut *p, Byte b);
};
#define IByteOut_Write(p, b) (p)->Write(p, b)


typedef struct ISeqInStream ISeqInStream;
struct ISeqInStream
{
SRes (*Read)(const ISeqInStream *p, void *buf, size_t *size);
/* if (input(*size) != 0 && output(*size) == 0) means end_of_stream.
(output(*size) < input(*size)) is allowed */
};
#define ISeqInStream_Read(p, buf, size) (p)->Read(p, buf, size)

/* it can return SZ_ERROR_INPUT_EOF */
SRes SeqInStream_Read(const ISeqInStream *stream, void *buf, size_t size);
SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes errorType);
SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf);


typedef struct ISeqOutStream ISeqOutStream;
struct ISeqOutStream
{
size_t (*Write)(const ISeqOutStream *p, const void *buf, size_t size);
/* Returns: result - the number of actually written bytes.
(result < size) means error */
};
#define ISeqOutStream_Write(p, buf, size) (p)->Write(p, buf, size)

typedef enum
{
SZ_SEEK_SET = 0,
SZ_SEEK_CUR = 1,
SZ_SEEK_END = 2
} ESzSeek;


typedef struct ISeekInStream ISeekInStream;
struct ISeekInStream
{
SRes (*Read)(const ISeekInStream *p, void *buf, size_t *size); /* same as ISeqInStream::Read */
SRes (*Seek)(const ISeekInStream *p, Int64 *pos, ESzSeek origin);
};
#define ISeekInStream_Read(p, buf, size) (p)->Read(p, buf, size)
#define ISeekInStream_Seek(p, pos, origin) (p)->Seek(p, pos, origin)


typedef struct ILookInStream ILookInStream;
struct ILookInStream
{
SRes (*Look)(const ILookInStream *p, const void **buf, size_t *size);
/* if (input(*size) != 0 && output(*size) == 0) means end_of_stream.
(output(*size) > input(*size)) is not allowed
(output(*size) < input(*size)) is allowed */
SRes (*Skip)(const ILookInStream *p, size_t offset);
/* offset must be <= output(*size) of Look */

SRes (*Read)(const ILookInStream *p, void *buf, size_t *size);
/* reads directly (without buffer). It's same as ISeqInStream::Read */
SRes (*Seek)(const ILookInStream *p, Int64 *pos, ESzSeek origin);
};

#define ILookInStream_Look(p, buf, size) (p)->Look(p, buf, size)
#define ILookInStream_Skip(p, offset) (p)->Skip(p, offset)
#define ILookInStream_Read(p, buf, size) (p)->Read(p, buf, size)
#define ILookInStream_Seek(p, pos, origin) (p)->Seek(p, pos, origin)


SRes LookInStream_LookRead(const ILookInStream *stream, void *buf, size_t *size);
SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset);

/* reads via ILookInStream::Read */
SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRes errorType);
SRes LookInStream_Read(const ILookInStream *stream, void *buf, size_t size);



typedef struct
{
ILookInStream vt;
const ISeekInStream *realStream;

size_t pos;
size_t size; /* it's data size */

/* the following variables must be set outside */
Byte *buf;
size_t bufSize;
} CLookToRead2;

void LookToRead2_CreateVTable(CLookToRead2 *p, int lookahead);

#define LookToRead2_Init(p) { (p)->pos = (p)->size = 0; }


typedef struct
{
ISeqInStream vt;
const ILookInStream *realStream;
} CSecToLook;

void SecToLook_CreateVTable(CSecToLook *p);



typedef struct
{
ISeqInStream vt;
const ILookInStream *realStream;
} CSecToRead;

void SecToRead_CreateVTable(CSecToRead *p);


typedef struct ICompressProgress ICompressProgress;

struct ICompressProgress
{
SRes (*Progress)(const ICompressProgress *p, UInt64 inSize, UInt64 outSize);
/* Returns: result. (result != SZ_OK) means break.
Value (UInt64)(Int64)-1 for size means unknown value. */
};
#define ICompressProgress_Progress(p, inSize, outSize) (p)->Progress(p, inSize, outSize)



typedef struct ISzAlloc ISzAlloc;
typedef const ISzAlloc * ISzAllocPtr;

struct ISzAlloc
{
void *(*Alloc)(ISzAllocPtr p, size_t size);
void (*Free)(ISzAllocPtr p, void *address); /* address can be 0 */
};

#define ISzAlloc_Alloc(p, size) (p)->Alloc(p, size)
#define ISzAlloc_Free(p, a) (p)->Free(p, a)

/* deprecated */
#define IAlloc_Alloc(p, size) ISzAlloc_Alloc(p, size)
#define IAlloc_Free(p, a) ISzAlloc_Free(p, a)





#ifndef MY_offsetof
#ifdef offsetof
#define MY_offsetof(type, m) offsetof(type, m)
/*
#define MY_offsetof(type, m) FIELD_OFFSET(type, m)
*/
#else
#define MY_offsetof(type, m) ((size_t)&(((type *)0)->m))
#endif
#endif



#ifndef MY_container_of

/*
#define MY_container_of(ptr, type, m) container_of(ptr, type, m)
#define MY_container_of(ptr, type, m) CONTAINING_RECORD(ptr, type, m)
#define MY_container_of(ptr, type, m) ((type *)((char *)(ptr) - offsetof(type, m)))
#define MY_container_of(ptr, type, m) (&((type *)0)->m == (ptr), ((type *)(((char *)(ptr)) - MY_offsetof(type, m))))
*/

/*
GCC shows warning: "perhaps the 'offsetof' macro was used incorrectly"
GCC 3.4.4 : classes with constructor
GCC 4.8.1 : classes with non-public variable members"
*/

#define MY_container_of(ptr, type, m) ((type *)((char *)(1 ? (ptr) : &((type *)0)->m) - MY_offsetof(type, m)))


#endif

#define CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) ((type *)(ptr))

/*
#define CONTAINER_FROM_VTBL(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m)
*/
#define CONTAINER_FROM_VTBL(ptr, type, m) MY_container_of(ptr, type, m)

#define CONTAINER_FROM_VTBL_CLS(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m)
/*
#define CONTAINER_FROM_VTBL_CLS(ptr, type, m) CONTAINER_FROM_VTBL(ptr, type, m)
*/



#ifdef _WIN32

#define CHAR_PATH_SEPARATOR '\\'
#define WCHAR_PATH_SEPARATOR L'\\'
#define STRING_PATH_SEPARATOR "\\"
#define WSTRING_PATH_SEPARATOR L"\\"

#else

#define CHAR_PATH_SEPARATOR '/'
#define WCHAR_PATH_SEPARATOR L'/'
#define STRING_PATH_SEPARATOR "/"
#define WSTRING_PATH_SEPARATOR L"/"

#endif

EXTERN_C_END

#endif
33 changes: 33 additions & 0 deletions htp/lzma/Compiler.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/* Compiler.h
2017-04-03 : Igor Pavlov : Public domain */

#ifndef __7Z_COMPILER_H
#define __7Z_COMPILER_H

#ifdef _MSC_VER

#ifdef UNDER_CE
#define RPC_NO_WINDOWS_H
/* #pragma warning(disable : 4115) // '_RPC_ASYNC_STATE' : named type definition in parentheses */
#pragma warning(disable : 4201) // nonstandard extension used : nameless struct/union
#pragma warning(disable : 4214) // nonstandard extension used : bit field types other than int
#endif

#if _MSC_VER >= 1300
#pragma warning(disable : 4996) // This function or variable may be unsafe
#else
#pragma warning(disable : 4511) // copy constructor could not be generated
#pragma warning(disable : 4512) // assignment operator could not be generated
#pragma warning(disable : 4514) // unreferenced inline function has been removed
#pragma warning(disable : 4702) // unreachable code
#pragma warning(disable : 4710) // not inlined
#pragma warning(disable : 4714) // function marked as __forceinline not inlined
#pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information
#endif

#endif

#define UNUSED_VAR(x) (void)x;
/* #define UNUSED_VAR(x) x=x; */

#endif
1,127 changes: 1,127 additions & 0 deletions htp/lzma/LzFind.c

Large diffs are not rendered by default.

121 changes: 121 additions & 0 deletions htp/lzma/LzFind.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/* LzFind.h -- Match finder for LZ algorithms
2017-06-10 : Igor Pavlov : Public domain */

#ifndef __LZ_FIND_H
#define __LZ_FIND_H

#include "7zTypes.h"

EXTERN_C_BEGIN

typedef UInt32 CLzRef;

typedef struct _CMatchFinder
{
Byte *buffer;
UInt32 pos;
UInt32 posLimit;
UInt32 streamPos;
UInt32 lenLimit;

UInt32 cyclicBufferPos;
UInt32 cyclicBufferSize; /* it must be = (historySize + 1) */

Byte streamEndWasReached;
Byte btMode;
Byte bigHash;
Byte directInput;

UInt32 matchMaxLen;
CLzRef *hash;
CLzRef *son;
UInt32 hashMask;
UInt32 cutValue;

Byte *bufferBase;
ISeqInStream *stream;

UInt32 blockSize;
UInt32 keepSizeBefore;
UInt32 keepSizeAfter;

UInt32 numHashBytes;
size_t directInputRem;
UInt32 historySize;
UInt32 fixedHashSize;
UInt32 hashSizeSum;
SRes result;
UInt32 crc[256];
size_t numRefs;

UInt64 expectedDataSize;
} CMatchFinder;

#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer)

#define Inline_MatchFinder_GetNumAvailableBytes(p) ((p)->streamPos - (p)->pos)

#define Inline_MatchFinder_IsFinishedOK(p) \
((p)->streamEndWasReached \
&& (p)->streamPos == (p)->pos \
&& (!(p)->directInput || (p)->directInputRem == 0))

int MatchFinder_NeedMove(CMatchFinder *p);
Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p);
void MatchFinder_MoveBlock(CMatchFinder *p);
void MatchFinder_ReadIfRequired(CMatchFinder *p);

void MatchFinder_Construct(CMatchFinder *p);

/* Conditions:
historySize <= 3 GB
keepAddBufferBefore + matchMaxLen + keepAddBufferAfter < 511MB
*/
int MatchFinder_Create(CMatchFinder *p, UInt32 historySize,
UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter,
ISzAllocPtr alloc);
void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc);
void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems);
void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue);

UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *buffer, CLzRef *son,
UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue,
UInt32 *distances, UInt32 maxLen);

/*
Conditions:
Mf_GetNumAvailableBytes_Func must be called before each Mf_GetMatchLen_Func.
Mf_GetPointerToCurrentPos_Func's result must be used only before any other function
*/

typedef void (*Mf_Init_Func)(void *object);
typedef UInt32 (*Mf_GetNumAvailableBytes_Func)(void *object);
typedef const Byte * (*Mf_GetPointerToCurrentPos_Func)(void *object);
typedef UInt32 (*Mf_GetMatches_Func)(void *object, UInt32 *distances);
typedef void (*Mf_Skip_Func)(void *object, UInt32);

typedef struct _IMatchFinder
{
Mf_Init_Func Init;
Mf_GetNumAvailableBytes_Func GetNumAvailableBytes;
Mf_GetPointerToCurrentPos_Func GetPointerToCurrentPos;
Mf_GetMatches_Func GetMatches;
Mf_Skip_Func Skip;
} IMatchFinder;

void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable);

void MatchFinder_Init_LowHash(CMatchFinder *p);
void MatchFinder_Init_HighHash(CMatchFinder *p);
void MatchFinder_Init_3(CMatchFinder *p, int readData);
void MatchFinder_Init(CMatchFinder *p);

UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances);
UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances);

void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num);
void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num);

EXTERN_C_END

#endif
57 changes: 57 additions & 0 deletions htp/lzma/LzHash.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/* LzHash.h -- HASH functions for LZ algorithms
2015-04-12 : Igor Pavlov : Public domain */

#ifndef __LZ_HASH_H
#define __LZ_HASH_H

#define kHash2Size (1 << 10)
#define kHash3Size (1 << 16)
#define kHash4Size (1 << 20)

#define kFix3HashSize (kHash2Size)
#define kFix4HashSize (kHash2Size + kHash3Size)
#define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size)

#define HASH2_CALC hv = cur[0] | ((UInt32)cur[1] << 8);

#define HASH3_CALC { \
UInt32 temp = p->crc[cur[0]] ^ cur[1]; \
h2 = temp & (kHash2Size - 1); \
hv = (temp ^ ((UInt32)cur[2] << 8)) & p->hashMask; }

#define HASH4_CALC { \
UInt32 temp = p->crc[cur[0]] ^ cur[1]; \
h2 = temp & (kHash2Size - 1); \
temp ^= ((UInt32)cur[2] << 8); \
h3 = temp & (kHash3Size - 1); \
hv = (temp ^ (p->crc[cur[3]] << 5)) & p->hashMask; }

#define HASH5_CALC { \
UInt32 temp = p->crc[cur[0]] ^ cur[1]; \
h2 = temp & (kHash2Size - 1); \
temp ^= ((UInt32)cur[2] << 8); \
h3 = temp & (kHash3Size - 1); \
temp ^= (p->crc[cur[3]] << 5); \
h4 = temp & (kHash4Size - 1); \
hv = (temp ^ (p->crc[cur[4]] << 3)) & p->hashMask; }

/* #define HASH_ZIP_CALC hv = ((cur[0] | ((UInt32)cur[1] << 8)) ^ p->crc[cur[2]]) & 0xFFFF; */
#define HASH_ZIP_CALC hv = ((cur[2] | ((UInt32)cur[0] << 8)) ^ p->crc[cur[1]]) & 0xFFFF;


#define MT_HASH2_CALC \
h2 = (p->crc[cur[0]] ^ cur[1]) & (kHash2Size - 1);

#define MT_HASH3_CALC { \
UInt32 temp = p->crc[cur[0]] ^ cur[1]; \
h2 = temp & (kHash2Size - 1); \
h3 = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); }

#define MT_HASH4_CALC { \
UInt32 temp = p->crc[cur[0]] ^ cur[1]; \
h2 = temp & (kHash2Size - 1); \
temp ^= ((UInt32)cur[2] << 8); \
h3 = temp & (kHash3Size - 1); \
h4 = (temp ^ (p->crc[cur[3]] << 5)) & (kHash4Size - 1); }

#endif
1,223 changes: 1,223 additions & 0 deletions htp/lzma/LzmaDec.c

Large diffs are not rendered by default.

234 changes: 234 additions & 0 deletions htp/lzma/LzmaDec.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,234 @@
/* LzmaDec.h -- LZMA Decoder
2018-04-21 : Igor Pavlov : Public domain */

#ifndef __LZMA_DEC_H
#define __LZMA_DEC_H

#include "7zTypes.h"

EXTERN_C_BEGIN

/* #define _LZMA_PROB32 */
/* _LZMA_PROB32 can increase the speed on some CPUs,
but memory usage for CLzmaDec::probs will be doubled in that case */

typedef
#ifdef _LZMA_PROB32
UInt32
#else
UInt16
#endif
CLzmaProb;


/* ---------- LZMA Properties ---------- */

#define LZMA_PROPS_SIZE 5

typedef struct _CLzmaProps
{
Byte lc;
Byte lp;
Byte pb;
Byte _pad_;
UInt32 dicSize;
} CLzmaProps;

/* LzmaProps_Decode - decodes properties
Returns:
SZ_OK
SZ_ERROR_UNSUPPORTED - Unsupported properties
*/

SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size);


/* ---------- LZMA Decoder state ---------- */

/* LZMA_REQUIRED_INPUT_MAX = number of required input bytes for worst case.
Num bits = log2((2^11 / 31) ^ 22) + 26 < 134 + 26 = 160; */

#define LZMA_REQUIRED_INPUT_MAX 20

typedef struct
{
/* Don't change this structure. ASM code can use it. */
CLzmaProps prop;
CLzmaProb *probs;
CLzmaProb *probs_1664;
Byte *dic;
SizeT dicBufSize;
SizeT dicPos;
const Byte *buf;
UInt32 range;
UInt32 code;
UInt32 processedPos;
UInt32 checkDicSize;
UInt32 reps[4];
UInt32 state;
UInt32 remainLen;

UInt32 numProbs;
unsigned tempBufSize;
Byte tempBuf[LZMA_REQUIRED_INPUT_MAX];
} CLzmaDec;

#define LzmaDec_Construct(p) { (p)->dic = NULL; (p)->probs = NULL; }

void LzmaDec_Init(CLzmaDec *p);

/* There are two types of LZMA streams:
- Stream with end mark. That end mark adds about 6 bytes to compressed size.
- Stream without end mark. You must know exact uncompressed size to decompress such stream. */

typedef enum
{
LZMA_FINISH_ANY, /* finish at any point */
LZMA_FINISH_END /* block must be finished at the end */
} ELzmaFinishMode;

/* ELzmaFinishMode has meaning only if the decoding reaches output limit !!!
You must use LZMA_FINISH_END, when you know that current output buffer
covers last bytes of block. In other cases you must use LZMA_FINISH_ANY.
If LZMA decoder sees end marker before reaching output limit, it returns SZ_OK,
and output value of destLen will be less than output buffer size limit.
You can check status result also.
You can use multiple checks to test data integrity after full decompression:
1) Check Result and "status" variable.
2) Check that output(destLen) = uncompressedSize, if you know real uncompressedSize.
3) Check that output(srcLen) = compressedSize, if you know real compressedSize.
You must use correct finish mode in that case. */

typedef enum
{
LZMA_STATUS_NOT_SPECIFIED, /* use main error code instead */
LZMA_STATUS_FINISHED_WITH_MARK, /* stream was finished with end mark. */
LZMA_STATUS_NOT_FINISHED, /* stream was not finished */
LZMA_STATUS_NEEDS_MORE_INPUT, /* you must provide more input bytes */
LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK /* there is probability that stream was finished without end mark */
} ELzmaStatus;

/* ELzmaStatus is used only as output value for function call */


/* ---------- Interfaces ---------- */

/* There are 3 levels of interfaces:
1) Dictionary Interface
2) Buffer Interface
3) One Call Interface
You can select any of these interfaces, but don't mix functions from different
groups for same object. */


/* There are two variants to allocate state for Dictionary Interface:
1) LzmaDec_Allocate / LzmaDec_Free
2) LzmaDec_AllocateProbs / LzmaDec_FreeProbs
You can use variant 2, if you set dictionary buffer manually.
For Buffer Interface you must always use variant 1.
LzmaDec_Allocate* can return:
SZ_OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_UNSUPPORTED - Unsupported properties
*/

SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc);
void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc);

SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc);
void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc);

/* ---------- Dictionary Interface ---------- */

/* You can use it, if you want to eliminate the overhead for data copying from
dictionary to some other external buffer.
You must work with CLzmaDec variables directly in this interface.
STEPS:
LzmaDec_Construct()
LzmaDec_Allocate()
for (each new stream)
{
LzmaDec_Init()
while (it needs more decompression)
{
LzmaDec_DecodeToDic()
use data from CLzmaDec::dic and update CLzmaDec::dicPos
}
}
LzmaDec_Free()
*/

/* LzmaDec_DecodeToDic
The decoding to internal dictionary buffer (CLzmaDec::dic).
You must manually update CLzmaDec::dicPos, if it reaches CLzmaDec::dicBufSize !!!
finishMode:
It has meaning only if the decoding reaches output limit (dicLimit).
LZMA_FINISH_ANY - Decode just dicLimit bytes.
LZMA_FINISH_END - Stream must be finished after dicLimit.
Returns:
SZ_OK
status:
LZMA_STATUS_FINISHED_WITH_MARK
LZMA_STATUS_NOT_FINISHED
LZMA_STATUS_NEEDS_MORE_INPUT
LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK
SZ_ERROR_DATA - Data error
*/

SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit,
const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status, SizeT memlimit);


/* ---------- Buffer Interface ---------- */

/* It's zlib-like interface.
See LzmaDec_DecodeToDic description for information about STEPS and return results,
but you must use LzmaDec_DecodeToBuf instead of LzmaDec_DecodeToDic and you don't need
to work with CLzmaDec variables manually.
finishMode:
It has meaning only if the decoding reaches output limit (*destLen).
LZMA_FINISH_ANY - Decode just destLen bytes.
LZMA_FINISH_END - Stream must be finished after (*destLen).
*/

SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen,
const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status, SizeT memlimit);


/* ---------- One Call Interface ---------- */

/* LzmaDecode
finishMode:
It has meaning only if the decoding reaches output limit (*destLen).
LZMA_FINISH_ANY - Decode just destLen bytes.
LZMA_FINISH_END - Stream must be finished after (*destLen).
Returns:
SZ_OK
status:
LZMA_STATUS_FINISHED_WITH_MARK
LZMA_STATUS_NOT_FINISHED
LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK
SZ_ERROR_DATA - Data error
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_UNSUPPORTED - Unsupported properties
SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src).
*/

SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen,
const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode,
ELzmaStatus *status, ISzAllocPtr alloc);

EXTERN_C_END

#endif
16 changes: 16 additions & 0 deletions htp/lzma/Makefile.am
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@

h_sources = LzmaDec.h 7zTypes.h

h_sources_private = LzFind.h LzHash.h Compiler.h Precomp.h

c_sources = LzFind.c LzmaDec.c

AM_CFLAGS = -I$(top_srcdir) -D_GNU_SOURCE -g -Wall -Wextra -std=gnu99 -pedantic \
-Wextra -Wno-missing-field-initializers -Wshadow -Wpointer-arith \
-Wstrict-prototypes -Wmissing-prototypes -Wno-unused-parameter

library_includedir = $(includedir)/$(GENERIC_LIBRARY_NAME)/lzma
library_include_HEADERS = $(h_sources)

noinst_LTLIBRARIES = liblzma-c.la
liblzma_c_la_SOURCES = $(h_sources) $(h_sources_private) $(c_sources)
10 changes: 10 additions & 0 deletions htp/lzma/Precomp.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/* Precomp.h -- StdAfx
2013-11-12 : Igor Pavlov : Public domain */

#ifndef __7Z_PRECOMP_H
#define __7Z_PRECOMP_H

#include "Compiler.h"
/* #include "7zTypes.h" */

#endif
File renamed without changes.
2 changes: 1 addition & 1 deletion test/files/88-response-multiple-cl-mismatch.t
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ Date: Mon, 31 Aug 2009 20:25:50 GMT
Server: Apache
Connection: close
Content-Type: text/html
Content-Length: 11
Content-Length: 12
Content-Length: 11

Hello World!
15 changes: 15 additions & 0 deletions test/files/91-request-unexpected-body.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
>>>
POST / HTTP/1.1
Host: localhost
Content-Type: application/x-www-form-urlencoded

login=foo&password=bar
<<<
HTTP/1.1 200 OK
Content-Length: 0


>>>
GET / HTTP/1.1
Host: localhost

3 changes: 3 additions & 0 deletions test/files/92-http_0_9-method_only.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
>>>
GET /

Binary file added test/files/93-compressed-response-deflateasgzip.t
Binary file not shown.
Binary file added test/files/94-compressed-response-multiple.t
Binary file not shown.
Binary file added test/files/95-compressed-response-gzipasdeflate.t
Binary file not shown.
Binary file added test/files/96-compressed-response-lzma.t
Binary file not shown.
13 changes: 13 additions & 0 deletions test/fuzz/fuzz_htp.c
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,18 @@ static int HTPCallbackResponse(htp_tx_t *out_tx) {
static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
{
fprintf(logfile, "HTPCallbackRequestHeaderData %"PRIuMAX"\n", (uintmax_t)tx_data->len);
if (tx_data->len > 0) {
fprintf(logfile, "HTPCallbackRequestHeaderData %x %x\n", tx_data->data[0], tx_data->data[(uintmax_t)tx_data->len-1]);
}
return 0;
}

static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
{
fprintf(logfile, "HTPCallbackResponseHeaderData %"PRIuMAX"\n", (uintmax_t)tx_data->len);
if (tx_data->len > 0) {
fprintf(logfile, "HTPCallbackResponseHeaderData %x %x\n", tx_data->data[0], tx_data->data[(uintmax_t)tx_data->len-1]);
}
return 0;
}

Expand All @@ -62,12 +68,18 @@ static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
static int HTPCallbackRequestBodyData(htp_tx_data_t *tx_data)
{
fprintf(logfile, "HTPCallbackRequestBodyData %"PRIuMAX"\n", (uintmax_t)tx_data->len);
if (tx_data->len > 0) {
fprintf(logfile, "HTPCallbackRequestBodyData %x %x\n", tx_data->data[0], tx_data->data[(uintmax_t)tx_data->len-1]);
}
return 0;
}

static int HTPCallbackResponseBodyData(htp_tx_data_t *tx_data)
{
fprintf(logfile, "HTPCallbackResponseBodyData %"PRIuMAX"\n", (uintmax_t)tx_data->len);
if (tx_data->len > 0) {
fprintf(logfile, "HTPCallbackResponseBodyData %x %x\n", tx_data->data[0], tx_data->data[(uintmax_t)tx_data->len-1]);
}
return 0;
}

Expand Down Expand Up @@ -220,6 +232,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
htp_connp_res_data(connp, NULL, out_data + out_data_offset, out_data_len - out_data_offset);
}

htp_connp_close(connp, NULL);
htp_connp_destroy_all(connp);
// Destroy LibHTP configuration
htp_config_destroy(cfg);
Expand Down
17 changes: 17 additions & 0 deletions test/pcaptohtp.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import sys
import binascii

# Transforms a pcap into a test file for libhtp
# tshark -Tfields -e tcp.dstport -e tcp.payload -r input.pcap > input.txt
# python pcaptohtp.py input.txt > input.t

f = open(sys.argv[1])
for l in f.readlines():
portAndPl=l.split()
if len(portAndPl) == 2:
# determine request or response based on port
if portAndPl[0] == "80":
print(">>>")
else:
print("<<<")
print(binascii.unhexlify(portAndPl[1].replace(":","")))
130 changes: 124 additions & 6 deletions test/test_main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1009,8 +1009,8 @@ TEST_F(ConnectionParsing, AuthDigest) {
ASSERT_TRUE(tx->request_auth_password == NULL);
}

TEST_F(ConnectionParsing, Http_0_9_MethodOnly) {
int rc = test_run(home, "42-http_0_9-method_only.t", cfg, &connp);
TEST_F(ConnectionParsing, Unknown_MethodOnly) {
int rc = test_run(home, "42-unknown-method_only.t", cfg, &connp);
ASSERT_GE(rc, 0);

htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0);
Expand Down Expand Up @@ -1668,28 +1668,43 @@ TEST_F(ConnectionParsing, ResponseMultipleCl) {

TEST_F(ConnectionParsing, ResponseMultipleClMismatch) {
int rc = test_run(home, "88-response-multiple-cl-mismatch.t", cfg, &connp);
ASSERT_LT(rc, 0); // Expect error
ASSERT_GE(rc, 0);

ASSERT_EQ(1, htp_list_size(connp->conn->transactions));

htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0);
ASSERT_TRUE(tx != NULL);

ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress);
ASSERT_EQ(HTP_RESPONSE_HEADERS, tx->response_progress);
ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress);

ASSERT_TRUE(tx->flags & HTP_REQUEST_SMUGGLING);

htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers, "Content-Length");
ASSERT_TRUE(h != NULL);
ASSERT_TRUE(h->value != NULL);
ASSERT_TRUE(h->flags & HTP_FIELD_REPEATED);

ASSERT_EQ(0, bstr_cmp_c(h->value, "12"));

ASSERT_EQ(2, htp_list_size(tx->conn->messages));
htp_log_t *log = (htp_log_t *) htp_list_get(tx->conn->messages, 1);
ASSERT_TRUE(log != NULL);
ASSERT_EQ(0, strcmp(log->msg, "Ambiguous response C-L value"));
ASSERT_EQ(HTP_LOG_WARNING, log->level);
}

TEST_F(ConnectionParsing, ResponseInvalidCl) {
int rc = test_run(home, "75-response-invalid-cl.t", cfg, &connp);
ASSERT_LT(rc, 0); // Expect error.
ASSERT_GE(rc, 0);

ASSERT_EQ(1, htp_list_size(connp->conn->transactions));

htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0);
ASSERT_TRUE(tx != NULL);

ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress);
ASSERT_EQ(HTP_RESPONSE_HEADERS, tx->response_progress);
ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress);

ASSERT_FALSE(tx->flags & HTP_REQUEST_SMUGGLING);
}
Expand Down Expand Up @@ -1914,3 +1929,106 @@ TEST_F(ConnectionParsing, RequestUriTooLarge) {
ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress);
ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress);
}

TEST_F(ConnectionParsing, RequestInvalid) {
int rc = test_run(home, "91-request-unexpected-body.t", cfg, &connp);
ASSERT_GE(rc, 0);

ASSERT_EQ(2, htp_list_size(connp->conn->transactions));

htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0);
ASSERT_TRUE(tx != NULL);
ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "POST"));
ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress);
ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress);

tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 1);
ASSERT_TRUE(tx != NULL);
ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "GET"));
ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress);
ASSERT_EQ(HTP_RESPONSE_NOT_STARTED, tx->response_progress);
}

TEST_F(ConnectionParsing, Http_0_9_MethodOnly) {
int rc = test_run(home, "92-http_0_9-method_only.t", cfg, &connp);
ASSERT_GE(rc, 0);

htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0);
ASSERT_TRUE(tx != NULL);

ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress);

ASSERT_TRUE(tx->request_method != NULL);
ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "GET"));

ASSERT_EQ(0, bstr_cmp_c(tx->request_uri, "/"));

ASSERT_EQ(1, tx->is_protocol_0_9);
}

TEST_F(ConnectionParsing, CompressedResponseDeflateAsGzip) {
int rc = test_run(home, "93-compressed-response-deflateasgzip.t", cfg, &connp);
ASSERT_GE(rc, 0);

ASSERT_EQ(1, htp_list_size(connp->conn->transactions));

htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0);
ASSERT_TRUE(tx != NULL);

ASSERT_TRUE(htp_tx_is_complete(tx));

ASSERT_EQ(755, tx->response_message_len);

ASSERT_EQ(1433, tx->response_entity_len);
}

TEST_F(ConnectionParsing, CompressedResponseMultiple) {
int rc = test_run(home, "94-compressed-response-multiple.t", cfg, &connp);
ASSERT_GE(rc, 0);

ASSERT_EQ(1, htp_list_size(connp->conn->transactions));

htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0);
ASSERT_TRUE(tx != NULL);

ASSERT_TRUE(htp_tx_is_complete(tx));

ASSERT_EQ(51, tx->response_message_len);

ASSERT_EQ(25, tx->response_entity_len);
}

TEST_F(ConnectionParsing, CompressedResponseGzipAsDeflate) {
int rc = test_run(home, "95-compressed-response-gzipasdeflate.t", cfg, &connp);
ASSERT_GE(rc, 0);

ASSERT_EQ(1, htp_list_size(connp->conn->transactions));

htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0);
ASSERT_TRUE(tx != NULL);

ASSERT_TRUE(htp_tx_is_complete(tx));

ASSERT_EQ(187, tx->response_message_len);

ASSERT_EQ(225, tx->response_entity_len);
}

#ifdef HAVE_LIBLZMA
TEST_F(ConnectionParsing, CompressedResponseLzma) {
int rc = test_run(home, "96-compressed-response-lzma.t", cfg, &connp);

ASSERT_GE(rc, 0);

ASSERT_EQ(1, htp_list_size(connp->conn->transactions));

htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0);
ASSERT_TRUE(tx != NULL);

ASSERT_TRUE(htp_tx_is_complete(tx));

ASSERT_EQ(90, tx->response_message_len);

ASSERT_EQ(68, tx->response_entity_len);
}
#endif
2 changes: 1 addition & 1 deletion test/test_utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ TEST(UtilTest, ParsePositiveIntegerWhitespace) {
TEST(UtilTest, ParseContentLength) {
bstr *str = bstr_dup_c("134");

EXPECT_EQ(134, htp_parse_content_length(str));
EXPECT_EQ(134, htp_parse_content_length(str, NULL));

bstr_free(str);
}
Expand Down