Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Printable/v16 #10592

Closed
wants to merge 14 commits into from
Closed

Printable/v16 #10592

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
cb0c12a
yaml: remove newline from error message
victorjulien Nov 20, 2023
d084856
stream: const args for StreamReassembleLog
victorjulien Nov 24, 2023
5e27718
eve/frame: implement payload-buffer-size option
victorjulien Nov 23, 2023
b32bda1
eve/alert: log payload directly from stream buffer
victorjulien Nov 20, 2023
20404c2
eve/alert: init membuffer size on missing config
victorjulien Nov 23, 2023
49523fc
eve/frames: pass membuffer to API
victorjulien Nov 21, 2023
d7c0d58
eve/frame: improve frame payload logging
victorjulien Nov 21, 2023
8ec62b4
unix-manager: add \n string to buffer using correct API call
victorjulien Nov 24, 2023
c2df1b0
membuffer: turn complex macros into functions
victorjulien Nov 24, 2023
bdbffed
membuffer: use buffer pointer as flexible array member
victorjulien Nov 24, 2023
40f9c7c
membuffer: return bytes written
victorjulien Nov 24, 2023
6e92e5d
eve/frame: break out of logging callback if buffer is full
victorjulien Nov 24, 2023
d38f9e1
eve/alert: break out of payload logging callback if buffer is full
victorjulien Nov 24, 2023
04204de
membuffer: annotate printf style function
victorjulien Nov 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/conf-yaml-loader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ static int ConfYamlParse(yaml_parser_t *parser, ConfNode *parent, int inseq, int

while (!done) {
if (!yaml_parser_parse(parser, &event)) {
SCLogError("Failed to parse configuration file at line %" PRIuMAX ": %s\n",
SCLogError("Failed to parse configuration file at line %" PRIuMAX ": %s",
(uintmax_t)parser->problem_mark.line, parser->problem);
retval = -1;
break;
Expand Down
115 changes: 69 additions & 46 deletions src/output-json-alert.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
#include "stream.h"
#include "threadvars.h"
#include "util-debug.h"
#include "stream-tcp.h"

#include "util-logopenfile.h"
#include "util-misc.h"
Expand Down Expand Up @@ -112,17 +113,6 @@ typedef struct JsonAlertLogThread_ {
OutputJsonThreadCtx *ctx;
} JsonAlertLogThread;

/* Callback function to pack payload contents from a stream into a buffer
* so we can report them in JSON output. */
static int AlertJsonDumpStreamSegmentCallback(
const Packet *p, TcpSegment *seg, void *data, const uint8_t *buf, uint32_t buflen)
{
MemBuffer *payload = (MemBuffer *)data;
MemBufferWriteRaw(payload, buf, buflen);

return 1;
}

static void AlertJsonSourceTarget(const Packet *p, const PacketAlert *pa,
JsonBuilder *js, JsonAddrInfo *addr)
{
Expand Down Expand Up @@ -414,7 +404,8 @@ static void AlertAddFiles(const Packet *p, JsonBuilder *jb, const uint64_t tx_id
}
}

static void AlertAddFrame(const Packet *p, JsonBuilder *jb, const int64_t frame_id)
static void AlertAddFrame(
const Packet *p, const int64_t frame_id, JsonBuilder *jb, MemBuffer *buffer)
{
if (p->flow == NULL || (p->proto == IPPROTO_TCP && p->flow->protoctx == NULL))
return;
Expand All @@ -436,7 +427,7 @@ static void AlertAddFrame(const Packet *p, JsonBuilder *jb, const int64_t frame_
}
Frame *frame = FrameGetById(frames, frame_id);
if (frame != NULL) {
FrameJsonLogOneFrame(IPPROTO_TCP, frame, p->flow, stream, p, jb);
FrameJsonLogOneFrame(IPPROTO_TCP, frame, p->flow, stream, p, jb, buffer);
}
} else if (p->proto == IPPROTO_UDP) {
if (PKT_IS_TOSERVER(p)) {
Expand All @@ -446,7 +437,7 @@ static void AlertAddFrame(const Packet *p, JsonBuilder *jb, const int64_t frame_
}
Frame *frame = FrameGetById(frames, frame_id);
if (frame != NULL) {
FrameJsonLogOneFrame(IPPROTO_UDP, frame, p->flow, NULL, p, jb);
FrameJsonLogOneFrame(IPPROTO_UDP, frame, p->flow, NULL, p, jb, buffer);
}
}
}
Expand Down Expand Up @@ -502,9 +493,64 @@ void EveAddVerdict(JsonBuilder *jb, const Packet *p)
jb_close(jb);
}

struct AlertJsonStreamDataCallbackData {
MemBuffer *payload;
uint64_t last_re;
};

static int AlertJsonStreamDataCallback(
void *cb_data, const uint8_t *input, const uint32_t input_len, const uint64_t input_offset)
{
struct AlertJsonStreamDataCallbackData *cbd = cb_data;
if (input_offset > cbd->last_re) {
MemBufferWriteString(
cbd->payload, "[%" PRIu64 " bytes missing]", input_offset - cbd->last_re);
}

int done = 0;
uint32_t written = MemBufferWriteRaw(cbd->payload, input, input_len);
if (written < input_len)
done = 1;
cbd->last_re = input_offset + input_len;
return done;
}

/** \internal
* \brief try to log stream data into payload/payload_printable
* \retval true stream data logged
* \retval false stream data not logged
*/
static bool AlertJsonStreamData(const AlertJsonOutputCtx *json_output_ctx, JsonAlertLogThread *aft,
Flow *f, const Packet *p, JsonBuilder *jb)
{
TcpSession *ssn = f->protoctx;
TcpStream *stream = (PKT_IS_TOSERVER(p)) ? &ssn->client : &ssn->server;

MemBufferReset(aft->payload_buffer);
struct AlertJsonStreamDataCallbackData cbd = { .payload = aft->payload_buffer,
.last_re = STREAM_BASE_OFFSET(stream) };
uint64_t unused = 0;
StreamReassembleLog(ssn, stream, AlertJsonStreamDataCallback, &cbd, STREAM_BASE_OFFSET(stream),
&unused, false);
if (cbd.payload->offset) {
if (json_output_ctx->flags & LOG_JSON_PAYLOAD_BASE64) {
jb_set_base64(jb, "payload", cbd.payload->buffer, cbd.payload->offset);
}

if (json_output_ctx->flags & LOG_JSON_PAYLOAD) {
uint8_t printable_buf[cbd.payload->offset + 1];
uint32_t offset = 0;
PrintStringsToBuffer(printable_buf, &offset, sizeof(printable_buf), cbd.payload->buffer,
cbd.payload->offset);
jb_set_string(jb, "payload_printable", (char *)printable_buf);
Comment on lines +541 to +545
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you think it would make sense to have a variant of jb_set_string that takes care of all this?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, that seems like a good idea. We could avoid this stack local copy, and perhaps it can be refactored to not even use the membuffer too? Would like to check that post merge though

}
return true;
}
return false;
}

static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
{
MemBuffer *payload = aft->payload_buffer;
AlertJsonOutputCtx *json_output_ctx = aft->json_output_ctx;

if (p->alerts.cnt == 0 && !(p->flags & PKT_HAS_TAG))
Expand Down Expand Up @@ -610,36 +656,14 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
int stream = (p->proto == IPPROTO_TCP) ?
(pa->flags & (PACKET_ALERT_FLAG_STATE_MATCH | PACKET_ALERT_FLAG_STREAM_MATCH) ?
1 : 0) : 0;
DEBUG_VALIDATE_BUG_ON(
p->flow == NULL); // should be impossible, but scan-build got confused

/* Is this a stream? If so, pack part of it into the payload field */
if (stream) {
uint8_t flag;

MemBufferReset(payload);

if (p->flowflags & FLOW_PKT_TOSERVER) {
flag = STREAM_DUMP_TOCLIENT;
} else {
flag = STREAM_DUMP_TOSERVER;
}

StreamSegmentForEach((const Packet *)p, flag,
AlertJsonDumpStreamSegmentCallback,
(void *)payload);
if (payload->offset) {
if (json_output_ctx->flags & LOG_JSON_PAYLOAD_BASE64) {
jb_set_base64(jb, "payload", payload->buffer, payload->offset);
}

if (json_output_ctx->flags & LOG_JSON_PAYLOAD) {
uint8_t printable_buf[payload->offset + 1];
uint32_t offset = 0;
PrintStringsToBuffer(printable_buf, &offset,
sizeof(printable_buf),
payload->buffer, payload->offset);
jb_set_string(jb, "payload_printable", (char *)printable_buf);
}
} else if (p->payload_len) {
if (stream && p->flow != NULL) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does it work when Suricata captures the flow midway and the alert wants to dump the payload on the first packet?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes there will be a flow then

const bool stream_data_logged =
AlertJsonStreamData(json_output_ctx, aft, p->flow, p, jb);
if (!stream_data_logged && p->payload_len) {
/* Fallback on packet payload */
AlertAddPayload(json_output_ctx, jb, p);
}
Expand All @@ -652,7 +676,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
}

if (pa->flags & PACKET_ALERT_FLAG_FRAME) {
AlertAddFrame(p, jb, pa->frame_id);
AlertAddFrame(p, pa->frame_id, jb, aft->payload_buffer);
}

/* base64-encoded full packet */
Expand Down Expand Up @@ -889,14 +913,13 @@ static void JsonAlertLogSetupMetadata(AlertJsonOutputCtx *json_output_ctx,
warn_no_meta = true;
}
}

json_output_ctx->payload_buffer_size = payload_buffer_size;
}

if (flags & LOG_JSON_RULE_METADATA) {
DetectEngineSetParseMetadata();
}

json_output_ctx->payload_buffer_size = payload_buffer_size;
json_output_ctx->flags |= flags;
}

Expand Down
135 changes: 89 additions & 46 deletions src/output-json-frame.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,61 +125,90 @@ static void PayloadAsHex(const uint8_t *data, uint32_t data_len, char *str, size
}
#endif

static void FrameAddPayloadTCP(JsonBuilder *js, const TcpStream *stream, const Frame *frame)
struct FrameJsonStreamDataCallbackData {
MemBuffer *payload;
const Frame *frame;
uint64_t last_re; /**< used to detect gaps */
};

static int FrameJsonStreamDataCallback(
void *cb_data, const uint8_t *input, const uint32_t input_len, const uint64_t input_offset)
{
uint32_t sb_data_len = 0;
const uint8_t *data = NULL;
uint64_t data_offset = 0;
struct FrameJsonStreamDataCallbackData *cbd = cb_data;
const Frame *frame = cbd->frame;

// TODO consider ACK'd
uint32_t write_size = input_len;
int done = 0;

if (frame->offset < STREAM_BASE_OFFSET(stream)) {
if (StreamingBufferGetData(&stream->sb, &data, &sb_data_len, &data_offset) == 0) {
SCLogDebug("NO DATA1");
return;
if (frame->len >= 0) {
const uint64_t data_re = input_offset + input_len;
const uint64_t frame_re = frame->offset + (uint64_t)frame->len;

/* data entirely after frame, we're done */
if (input_offset >= frame_re) {
return 1;
}
} else {
data_offset = (uint64_t)frame->offset;
SCLogDebug("data_offset %" PRIu64, data_offset);
if (StreamingBufferGetDataAtOffset(
&stream->sb, &data, &sb_data_len, (uint64_t)data_offset) == 0) {
SCLogDebug("NO DATA1");
return;
/* make sure to only log data belonging to the frame */
if (data_re >= frame_re) {
const uint64_t to_write = frame_re - input_offset;
if (to_write < (uint64_t)write_size) {
write_size = (uint32_t)to_write;
}
done = 1;
}
}
if (data == NULL || sb_data_len == 0) {
SCLogDebug("NO DATA2");
return;
if (input_offset > cbd->last_re) {
MemBufferWriteString(
cbd->payload, "[%" PRIu64 " bytes missing]", input_offset - cbd->last_re);
}

if (frame->len >= 0) {
sb_data_len = MIN(frame->len, (int32_t)sb_data_len);
if (write_size > 0) {
uint32_t written = MemBufferWriteRaw(cbd->payload, input, write_size);
if (written < write_size)
done = 1;
}
SCLogDebug("frame data_offset %" PRIu64 ", data_len %u frame len %" PRIi64, data_offset,
sb_data_len, frame->len);
cbd->last_re = input_offset + write_size;
return done;
}

/** \internal
* \brief try to log frame's stream data into payload/payload_printable
*/
static void FrameAddPayloadTCP(Flow *f, const TcpSession *ssn, const TcpStream *stream,
const Frame *frame, JsonBuilder *jb, MemBuffer *buffer)
{
MemBufferReset(buffer);

/* consider all data, ACK'd and non-ACK'd */
const uint64_t stream_data_re = StreamDataRightEdge(stream, true);
bool complete = false;
if (frame->len > 0) {
const uint64_t frame_re = frame->offset + (uint64_t)frame->len;
const uint64_t data_re = data_offset + sb_data_len;
complete = frame_re <= data_re;
if (frame->len >= 0 && frame->offset + (uint64_t)frame->len <= stream_data_re) {
complete = true;
}
jb_set_bool(js, "complete", complete);

uint32_t data_len = MIN(sb_data_len, 256);
jb_set_base64(js, "payload", data, data_len);
struct FrameJsonStreamDataCallbackData cbd = {
.payload = buffer, .frame = frame, .last_re = frame->offset
};
uint64_t unused = 0;
StreamReassembleLog(
ssn, stream, FrameJsonStreamDataCallback, &cbd, frame->offset, &unused, false);
/* if we have all data, but didn't log until the end of the frame, we have a gap at the
* end of the frame
* TODO what about not logging due to buffer full? */
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is todo intended to be left out?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not sure

if (complete && frame->len >= 0 && cbd.last_re < frame->offset + (uint64_t)frame->len) {
MemBufferWriteString(cbd.payload, "[%" PRIu64 " bytes missing]",
(frame->offset + (uint64_t)frame->len) - cbd.last_re);
}

uint8_t printable_buf[data_len + 1];
uint32_t o = 0;
PrintStringsToBuffer(printable_buf, &o, data_len + 1, data, data_len);
printable_buf[data_len] = '\0';
jb_set_string(js, "payload_printable", (char *)printable_buf);
#if 0
char pretty_buf[data_len * 4 + 1];
pretty_buf[0] = '\0';
PayloadAsHex(data, data_len, pretty_buf, data_len * 4 + 1);
jb_set_string(js, "payload_hex", pretty_buf);
#endif
if (cbd.payload->offset) {
jb_set_base64(jb, "payload", cbd.payload->buffer, cbd.payload->offset);
uint8_t printable_buf[cbd.payload->offset + 1];
uint32_t offset = 0;
PrintStringsToBuffer(printable_buf, &offset, sizeof(printable_buf), cbd.payload->buffer,
cbd.payload->offset);
jb_set_string(jb, "payload_printable", (char *)printable_buf);
jb_set_bool(jb, "complete", complete);
}
}

static void FrameAddPayloadUDP(JsonBuilder *js, const Packet *p, const Frame *frame)
Expand Down Expand Up @@ -223,8 +252,8 @@ static void FrameAddPayloadUDP(JsonBuilder *js, const Packet *p, const Frame *fr
/** \brief log a single frame
* \note ipproto argument is passed to assist static code analyzers
*/
void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, const Flow *f,
const TcpStream *stream, const Packet *p, JsonBuilder *jb)
void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, Flow *f,
const TcpStream *stream, const Packet *p, JsonBuilder *jb, MemBuffer *buffer)
{
DEBUG_VALIDATE_BUG_ON(ipproto != p->proto);
DEBUG_VALIDATE_BUG_ON(ipproto != f->proto);
Expand All @@ -249,7 +278,7 @@ void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, const Flow
} else {
jb_set_uint(jb, "length", frame->len);
}
FrameAddPayloadTCP(jb, stream, frame);
FrameAddPayloadTCP(f, f->protoctx, stream, frame, jb, buffer);
} else {
jb_set_uint(jb, "length", frame->len);
FrameAddPayloadUDP(jb, p, frame);
Expand Down Expand Up @@ -287,7 +316,7 @@ static int FrameJsonUdp(
return TM_ECODE_OK;

jb_set_string(jb, "app_proto", AppProtoToString(f->alproto));
FrameJsonLogOneFrame(IPPROTO_UDP, frame, p->flow, NULL, p, jb);
FrameJsonLogOneFrame(IPPROTO_UDP, frame, p->flow, NULL, p, jb, aft->payload_buffer);
OutputJsonBuilderBuffer(jb, aft->ctx);
jb_free(jb);
frame->flags |= FRAME_FLAG_LOGGED;
Expand Down Expand Up @@ -359,7 +388,7 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p)
return TM_ECODE_OK;

jb_set_string(jb, "app_proto", AppProtoToString(p->flow->alproto));
FrameJsonLogOneFrame(IPPROTO_TCP, frame, p->flow, stream, p, jb);
FrameJsonLogOneFrame(IPPROTO_TCP, frame, p->flow, stream, p, jb, aft->payload_buffer);
OutputJsonBuilderBuffer(jb, aft->ctx);
jb_free(jb);
frame->flags |= FRAME_FLAG_LOGGED;
Expand Down Expand Up @@ -482,8 +511,22 @@ static OutputInitResult JsonFrameLogInitCtxSub(ConfNode *conf, OutputCtx *parent
goto error;
}

uint32_t payload_buffer_size = 4096;
if (conf != NULL) {
const char *payload_buffer_value = ConfNodeLookupChildValue(conf, "payload-buffer-size");
if (payload_buffer_value != NULL) {
uint32_t value;
if (ParseSizeStringU32(payload_buffer_value, &value) < 0) {
SCLogError("Error parsing payload-buffer-size \"%s\"", payload_buffer_value);
goto error;
}
payload_buffer_size = value;
}
}

json_output_ctx->file_ctx = ajt->file_ctx;
json_output_ctx->eve_ctx = ajt;
json_output_ctx->payload_buffer_size = payload_buffer_size;

output_ctx->data = json_output_ctx;
output_ctx->DeInit = JsonFrameLogDeInitCtxSub;
Expand Down