diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 7dac3fcf0db..d4a8fd339f4 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -70,7 +70,7 @@ jobs: texlive-upquote \ texlive-capt-of \ texlive-needspace \ - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Bundling libhtp run: git clone https://github.com/OISF/libhtp -b 0.5.x - name: Bundling suricata-update diff --git a/ChangeLog b/ChangeLog index cbfa2554dd6..f5c63bd7dc8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,16 @@ +5.0.2 -- 2020-02-13 + +Bug #2993: Suricata 5.0.0beta1 memory allocation of 4294966034 bytes failed +Bug #3380: Segfault when using multi-detect +Bug #3400: smb: post-GAP file tx handling +Bug #3424: nfs: post-GAP some transactions never close +Bug #3425: nfs: post-GAP file tx handling +Bug #3433: coverity: CID 1456679: Memory - corruptions (NEGATIVE_RETURNS) +Bug #3434: coverity: CID 1456680: Incorrect expression (IDENTICAL_BRANCHES) +Bug #3469: gcc10: compilation failure unless -fcommon is supplied (5.0.x) +Bug #3473: Dropping privileges does not work with NFLOG (5.0.x) +Documentation #3423: readthedocs shows title of documentation as "Suricata unknown documentation" + 5.0.1 -- 2019-12-13 Bug #1871: intermittent abort()s at shutdown and in unix-socket diff --git a/configure.ac b/configure.ac index 8ca1f6744e5..3988017e4a3 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ - AC_INIT([suricata],[5.0.2-dev]) + AC_INIT([suricata],[5.0.3-dev]) m4_ifndef([AM_SILENT_RULES], [m4_define([AM_SILENT_RULES],[])])AM_SILENT_RULES([yes]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_SRCDIR([src/suricata.c]) diff --git a/doc/userguide/performance/tuning-considerations.rst b/doc/userguide/performance/tuning-considerations.rst index 78bd6e0860d..82caede1c7d 100644 --- a/doc/userguide/performance/tuning-considerations.rst +++ b/doc/userguide/performance/tuning-considerations.rst @@ -20,10 +20,10 @@ The memory is set up at start and the usage is as follows: mpm-algo: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Controls the pattern matcher algorithm. AC (``Aho–Corasick ``) is the default. +Controls the pattern matcher algorithm. AC (``Aho–Corasick``) is the default. On supported platforms, :doc:`hyperscan` is the best option. On commodity hardware if Hyperscan is not available the suggested setting is -``mpm-algo: ac-ks`` (``Aho–Corasick `` Ken Steele variant) as it performs better than +``mpm-algo: ac-ks`` (``Aho–Corasick`` Ken Steele variant) as it performs better than ``mpm-algo: ac`` detect.profile: diff --git a/doc/userguide/rules/header-keywords.rst b/doc/userguide/rules/header-keywords.rst index e04f5119cfa..f8104ec61e5 100644 --- a/doc/userguide/rules/header-keywords.rst +++ b/doc/userguide/rules/header-keywords.rst @@ -111,6 +111,25 @@ The named variant of that example would be:: ip_proto:PIM +ipv4.hdr +^^^^^^^^ + +Sticky buffer to match on the whole IPv4 header. + +Example rule: + +.. container:: example-rule + + alert ip any any -> any any (:example-rule-emphasis:`ipv4.hdr; content:"|3A|"; offset:9; depth:1;` sid:1234; rev:5;) + +This example looks if byte 9 of IPv4 header has value 3A. +That means that the IPv4 protocol is ICMPv6. + +ipv6.hdr +^^^^^^^^ + +Sticky buffer to match on the whole IPv6 header. + id ^^ diff --git a/doc/userguide/rules/payload-keywords.rst b/doc/userguide/rules/payload-keywords.rst index ad4e5814af0..bee555c5231 100644 --- a/doc/userguide/rules/payload-keywords.rst +++ b/doc/userguide/rules/payload-keywords.rst @@ -304,7 +304,7 @@ The ``byte_test`` keyword extracts ```` and performs an operation Format:: byte_test:, [!], , [,relative] \ - [,][, string, ][, dce][, bitmask ]; + [,][, string, ][, dce]; +----------------+------------------------------------------------------------------------------+ @@ -337,8 +337,6 @@ Format:: +----------------+------------------------------------------------------------------------------+ | [dce] | Allow the DCE module determine the byte order | +----------------+------------------------------------------------------------------------------+ -| [bitmask] | Applies the AND operator on the bytes converted | -+----------------+------------------------------------------------------------------------------+ Example:: @@ -376,7 +374,7 @@ Format:: byte_jump:, [, relative][, multiplier ] \ [, ][, string, ][, align][, from_beginning][, from_end] \ - [, post_offset ][, dce][, bitmask ]; + [, post_offset ][, dce]; +-----------------------+-----------------------------------------------------------------------+ | | The number of bytes selected from the packet to be converted | @@ -408,9 +406,6 @@ Format:: +-----------------------+-----------------------------------------------------------------------+ | [dce] | Allow the DCE module determine the byte order | +-----------------------+-----------------------------------------------------------------------+ -| [bitmask] | The AND operator will be applied by and the | -| | converted bytes, then jump operation is performed | -+-----------------------+-----------------------------------------------------------------------+ Example:: diff --git a/rules/smtp-events.rules b/rules/smtp-events.rules index 89c25acbc73..cc7eedcbbf6 100644 --- a/rules/smtp-events.rules +++ b/rules/smtp-events.rules @@ -29,4 +29,5 @@ alert smtp any any -> any any (msg:"SURICATA SMTP Mime boundary length exceeded" alert smtp any any -> any any (msg:"SURICATA SMTP duplicate fields"; flow:established,to_server; app-layer-event:smtp.duplicate_fields; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220018; rev:1;) alert smtp any any -> any any (msg:"SURICATA SMTP unparsable content"; flow:established,to_server; app-layer-event:smtp.unparsable_content; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220019; rev:1;) -# next sid 2220020 +alert smtp any any -> any any (msg:"SURICATA SMTP filename truncated"; flow:established,to_server; app-layer-event:smtp.mime_long_filename; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220020; rev:1;) +# next sid 2220021 diff --git a/rust/src/core.rs b/rust/src/core.rs index 06dda376aa1..ef892e66e8d 100644 --- a/rust/src/core.rs +++ b/rust/src/core.rs @@ -21,7 +21,6 @@ use std; use crate::filecontainer::*; /// Opaque C types. -pub enum Flow {} pub enum DetectEngineState {} pub enum AppLayerDecoderEvents {} @@ -183,3 +182,26 @@ pub fn sc_app_layer_decoder_events_free_events( } } } + +/// Opaque flow type (defined in C) +pub enum Flow {} + +/// Extern functions operating on Flow. +extern { + pub fn FlowGetLastTimeAsParts(flow: &Flow, secs: *mut u64, usecs: *mut u64); +} + +/// Rust implementation of Flow. +impl Flow { + + /// Return the time of the last flow update as a `Duration` + /// since the epoch. + pub fn get_last_time(&mut self) -> std::time::Duration { + unsafe { + let mut secs: u64 = 0; + let mut usecs: u64 = 0; + FlowGetLastTimeAsParts(self, &mut secs, &mut usecs); + std::time::Duration::new(secs, usecs as u32 * 1000) + } + } +} \ No newline at end of file diff --git a/rust/src/nfs/nfs.rs b/rust/src/nfs/nfs.rs index 4cc88920e36..365f9499071 100644 --- a/rust/src/nfs/nfs.rs +++ b/rust/src/nfs/nfs.rs @@ -119,6 +119,10 @@ pub struct NFSTransactionFile { /// last xid of this file transfer. Last READ or COMMIT normally. pub file_last_xid: u32, + /// after a gap, this will be set to a time in the future. If the file + /// receives no updates before that, it will be considered complete. + pub post_gap_ts: u64, + /// file tracker for a single file. Boxed so that we don't use /// as much space if we're not a file tx. pub file_tracker: FileTransferTracker, @@ -130,6 +134,7 @@ impl NFSTransactionFile { file_additional_procs: Vec::new(), chunk_count:0, file_last_xid: 0, + post_gap_ts: 0, file_tracker: FileTransferTracker::new(), } } @@ -338,12 +343,20 @@ pub struct NFSState { is_udp: bool, + /// true as long as we have file txs that are in a post-gap + /// state. It means we'll do extra house keeping for those. + check_post_gap_file_txs: bool, + pub nfs_version: u16, pub events: u16, /// tx counter for assigning incrementing id's to tx's tx_id: u64, + + /// Timestamp in seconds of last update. This is packet time, + /// potentially coming from pcaps. + ts: u64, } impl NFSState { @@ -366,9 +379,11 @@ impl NFSState { ts_gap:false, tc_gap:false, is_udp:false, + check_post_gap_file_txs:false, nfs_version:0, events:0, tx_id:0, + ts: 0, } } pub fn free(&mut self) { @@ -483,6 +498,71 @@ impl NFSState { } } + fn post_gap_housekeeping_for_files(&mut self) + { + let mut post_gap_txs = false; + for tx in &mut self.transactions { + if let Some(NFSTransactionTypeData::FILE(ref f)) = tx.type_data { + if f.post_gap_ts > 0 { + if self.ts > f.post_gap_ts { + tx.request_done = true; + tx.response_done = true; + } else { + post_gap_txs = true; + } + } + } + } + self.check_post_gap_file_txs = post_gap_txs; + } + + /* after a gap we will consider all transactions complete for our + * direction. File transfer transactions are an exception. Those + * can handle gaps. For the file transactions we set the current + * (flow) time and prune them in 60 seconds if no update for them + * was received. */ + fn post_gap_housekeeping(&mut self, dir: u8) + { + if self.ts_ssn_gap && dir == STREAM_TOSERVER { + for tx in &mut self.transactions { + if tx.id >= self.tx_id { + SCLogDebug!("post_gap_housekeeping: done"); + break; + } + if let Some(NFSTransactionTypeData::FILE(ref mut f)) = tx.type_data { + // leaving FILE txs open as they can deal with gaps. We + // remove them after 60 seconds of no activity though. + if f.post_gap_ts == 0 { + f.post_gap_ts = self.ts + 60; + self.check_post_gap_file_txs = true; + } + } else { + SCLogDebug!("post_gap_housekeeping: tx {} marked as done TS", tx.id); + tx.request_done = true; + } + } + } else if self.tc_ssn_gap && dir == STREAM_TOCLIENT { + for tx in &mut self.transactions { + if tx.id >= self.tx_id { + SCLogDebug!("post_gap_housekeeping: done"); + break; + } + if let Some(NFSTransactionTypeData::FILE(ref mut f)) = tx.type_data { + // leaving FILE txs open as they can deal with gaps. We + // remove them after 60 seconds of no activity though. + if f.post_gap_ts == 0 { + f.post_gap_ts = self.ts + 60; + self.check_post_gap_file_txs = true; + } + } else { + SCLogDebug!("post_gap_housekeeping: tx {} marked as done TC", tx.id); + tx.request_done = true; + tx.response_done = true; + } + } + } + } + pub fn process_request_record_lookup<'b>(&mut self, r: &RpcPacket<'b>, xidmap: &mut NFSRequestXidMap) { match parse_nfs3_request_lookup(r.prog_data) { Ok((_, lookup)) => { @@ -764,6 +844,11 @@ impl NFSState { } } + // reset timestamp if we get called after a gap + if tdf.post_gap_ts > 0 { + tdf.post_gap_ts = 0; + } + tdf.chunk_count += 1; let cs = tdf.file_tracker.update(files, flags, data, gap_size); /* see if we need to close the tx */ @@ -1113,6 +1198,12 @@ impl NFSState { }, } }; + + self.post_gap_housekeeping(STREAM_TOSERVER); + if self.check_post_gap_file_txs { + self.post_gap_housekeeping_for_files(); + } + status } @@ -1271,6 +1362,10 @@ impl NFSState { }, } }; + self.post_gap_housekeeping(STREAM_TOCLIENT); + if self.check_post_gap_file_txs { + self.post_gap_housekeeping_for_files(); + } status } /// Parsing function @@ -1357,7 +1452,7 @@ pub extern "C" fn rs_nfs_state_free(state: *mut std::os::raw::c_void) { /// C binding parse a NFS TCP request. Returns 1 on success, -1 on failure. #[no_mangle] -pub extern "C" fn rs_nfs_parse_request(_flow: *mut Flow, +pub extern "C" fn rs_nfs_parse_request(flow: &mut Flow, state: &mut NFSState, _pstate: *mut std::os::raw::c_void, input: *const u8, @@ -1368,6 +1463,7 @@ pub extern "C" fn rs_nfs_parse_request(_flow: *mut Flow, let buf = unsafe{std::slice::from_raw_parts(input, input_len as usize)}; SCLogDebug!("parsing {} bytes of request data", input_len); + state.ts = flow.get_last_time().as_secs(); if state.parse_tcp_data_ts(buf) == 0 { 1 } else { @@ -1388,7 +1484,7 @@ pub extern "C" fn rs_nfs_parse_request_tcp_gap( } #[no_mangle] -pub extern "C" fn rs_nfs_parse_response(_flow: *mut Flow, +pub extern "C" fn rs_nfs_parse_response(flow: &mut Flow, state: &mut NFSState, _pstate: *mut std::os::raw::c_void, input: *const u8, @@ -1399,6 +1495,7 @@ pub extern "C" fn rs_nfs_parse_response(_flow: *mut Flow, SCLogDebug!("parsing {} bytes of response data", input_len); let buf = unsafe{std::slice::from_raw_parts(input, input_len as usize)}; + state.ts = flow.get_last_time().as_secs(); if state.parse_tcp_data_tc(buf) == 0 { 1 } else { diff --git a/rust/src/nfs/rpc_records.rs b/rust/src/nfs/rpc_records.rs index 446935a0ba6..723c73e55d1 100644 --- a/rust/src/nfs/rpc_records.rs +++ b/rust/src/nfs/rpc_records.rs @@ -359,3 +359,23 @@ named!(pub parse_rpc_udp_reply, } )) ); + +#[cfg(test)] +mod tests { + use crate::nfs::rpc_records::*; + use nom::Err::Incomplete; + use nom::Needed::Size; + + #[test] + fn test_parse_input_too_short() { + let buf: &[u8] = &[ + 0x80, 0x0, 0x0, 0x9c, 0x8e, 0x28, 0x2, 0x7e + ]; + let r = parse_rpc_request_partial(buf); + match r { + Err(Incomplete(e)) => { assert_eq!(e, Size(4)); }, + _ => { panic!("failed {:?}",r); } + } + } +} + diff --git a/rust/src/smb/files.rs b/rust/src/smb/files.rs index 29f051fb99f..b8667e3fb87 100644 --- a/rust/src/smb/files.rs +++ b/rust/src/smb/files.rs @@ -30,6 +30,9 @@ pub struct SMBTransactionFile { pub file_name: Vec, pub share_name: Vec, pub file_tracker: FileTransferTracker, + /// after a gap, this will be set to a time in the future. If the file + /// receives no updates before that, it will be considered complete. + pub post_gap_ts: u64, } impl SMBTransactionFile { @@ -40,6 +43,7 @@ impl SMBTransactionFile { file_name: Vec::new(), share_name: Vec::new(), file_tracker: FileTransferTracker::new(), + post_gap_ts: 0, } } } @@ -201,6 +205,11 @@ impl SMBState { } } + // reset timestamp if we get called after a gap + if tdf.post_gap_ts > 0 { + tdf.post_gap_ts = 0; + } + let file_data = &data[0..data_to_handle_len]; let cs = tdf.file_tracker.update(files, flags, file_data, gap_size); cs diff --git a/rust/src/smb/smb.rs b/rust/src/smb/smb.rs index b74bf2e9146..2939108dd51 100644 --- a/rust/src/smb/smb.rs +++ b/rust/src/smb/smb.rs @@ -1,4 +1,4 @@ -/* Copyright (C) 2017 Open Information Security Foundation +/* Copyright (C) 2017-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -784,6 +784,10 @@ pub struct SMBState<> { pub ts_trunc: bool, // no more data for TOSERVER pub tc_trunc: bool, // no more data for TOCLIENT + /// true as long as we have file txs that are in a post-gap + /// state. It means we'll do extra house keeping for those. + check_post_gap_file_txs: bool, + /// transactions list pub transactions: Vec, @@ -798,6 +802,10 @@ pub struct SMBState<> { /// dcerpc interfaces, stored here to be able to match /// them while inspecting DCERPC REQUEST txs pub dcerpc_ifaces: Option>, + + /// Timestamp in seconds of last update. This is packet time, + /// potentially coming from pcaps. + ts: u64, } impl SMBState { @@ -824,11 +832,13 @@ impl SMBState { tc_gap: false, ts_trunc: false, tc_trunc: false, + check_post_gap_file_txs: false, transactions: Vec::new(), tx_id:0, dialect:0, dialect_vec: None, dcerpc_ifaces: None, + ts: 0, } } @@ -1143,9 +1153,29 @@ impl SMBState { (&name, is_dcerpc) } + fn post_gap_housekeeping_for_files(&mut self) + { + let mut post_gap_txs = false; + for tx in &mut self.transactions { + if let Some(SMBTransactionTypeData::FILE(ref f)) = tx.type_data { + if f.post_gap_ts > 0 { + if self.ts > f.post_gap_ts { + tx.request_done = true; + tx.response_done = true; + } else { + post_gap_txs = true; + } + } + } + } + self.check_post_gap_file_txs = post_gap_txs; + } + /* after a gap we will consider all transactions complete for our * direction. File transfer transactions are an exception. Those - * can handle gaps. */ + * can handle gaps. For the file transactions we set the current + * (flow) time and prune them in 60 seconds if no update for them + * was received. */ fn post_gap_housekeeping(&mut self, dir: u8) { if self.ts_ssn_gap && dir == STREAM_TOSERVER { @@ -1154,8 +1184,13 @@ impl SMBState { SCLogDebug!("post_gap_housekeeping: done"); break; } - if let Some(SMBTransactionTypeData::FILE(_)) = tx.type_data { - // leaving FILE txs open as they can deal with gaps. + if let Some(SMBTransactionTypeData::FILE(ref mut f)) = tx.type_data { + // leaving FILE txs open as they can deal with gaps. We + // remove them after 60 seconds of no activity though. + if f.post_gap_ts == 0 { + f.post_gap_ts = self.ts + 60; + self.check_post_gap_file_txs = true; + } } else { SCLogDebug!("post_gap_housekeeping: tx {} marked as done TS", tx.id); tx.request_done = true; @@ -1167,8 +1202,13 @@ impl SMBState { SCLogDebug!("post_gap_housekeeping: done"); break; } - if let Some(SMBTransactionTypeData::FILE(_)) = tx.type_data { - // leaving FILE txs open as they can deal with gaps. + if let Some(SMBTransactionTypeData::FILE(ref mut f)) = tx.type_data { + // leaving FILE txs open as they can deal with gaps. We + // remove them after 60 seconds of no activity though. + if f.post_gap_ts == 0 { + f.post_gap_ts = self.ts + 60; + self.check_post_gap_file_txs = true; + } } else { SCLogDebug!("post_gap_housekeeping: tx {} marked as done TC", tx.id); tx.request_done = true; @@ -1457,6 +1497,9 @@ impl SMBState { }; self.post_gap_housekeeping(STREAM_TOSERVER); + if self.check_post_gap_file_txs { + self.post_gap_housekeeping_for_files(); + } 0 } @@ -1684,6 +1727,9 @@ impl SMBState { } }; self.post_gap_housekeeping(STREAM_TOCLIENT); + if self.check_post_gap_file_txs { + self.post_gap_housekeeping_for_files(); + } self._debug_tx_stats(); 0 } @@ -1783,7 +1829,7 @@ pub extern "C" fn rs_smb_state_free(state: *mut std::os::raw::c_void) { /// C binding parse a SMB request. Returns 1 on success, -1 on failure. #[no_mangle] -pub extern "C" fn rs_smb_parse_request_tcp(_flow: *mut Flow, +pub extern "C" fn rs_smb_parse_request_tcp(flow: &mut Flow, state: &mut SMBState, _pstate: *mut std::os::raw::c_void, input: *const u8, @@ -1800,6 +1846,7 @@ pub extern "C" fn rs_smb_parse_request_tcp(_flow: *mut Flow, state.ts_gap = true; } + state.ts = flow.get_last_time().as_secs(); if state.parse_tcp_data_ts(buf) == 0 { return 1; } else { @@ -1821,7 +1868,7 @@ pub extern "C" fn rs_smb_parse_request_tcp_gap( #[no_mangle] -pub extern "C" fn rs_smb_parse_response_tcp(_flow: *mut Flow, +pub extern "C" fn rs_smb_parse_response_tcp(flow: &mut Flow, state: &mut SMBState, _pstate: *mut std::os::raw::c_void, input: *const u8, @@ -1838,6 +1885,7 @@ pub extern "C" fn rs_smb_parse_response_tcp(_flow: *mut Flow, state.tc_gap = true; } + state.ts = flow.get_last_time().as_secs(); if state.parse_tcp_data_tc(buf) == 0 { return 1; } else { diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 67da98189ff..3293a74e8aa 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -87,6 +87,8 @@ static SCRadixTree *cfgtree; /** List of HTP configurations. */ static HTPCfgRec cfglist; +SC_ATOMIC_DECLARE(uint32_t, htp_config_flags); + #ifdef DEBUG static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER; static uint64_t htp_state_memuse = 0; diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h index cdb6e99551a..b83b8c914da 100644 --- a/src/app-layer-htp.h +++ b/src/app-layer-htp.h @@ -271,7 +271,7 @@ typedef struct HtpState_ { /** part of the engine needs the request body (e.g. file_data keyword) */ #define HTP_REQUIRE_RESPONSE_BODY (1 << 3) -SC_ATOMIC_DECLARE(uint32_t, htp_config_flags); +SC_ATOMIC_EXTERN(uint32_t, htp_config_flags); void RegisterHTPParsers(void); void HTPParserRegisterTests(void); diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index 347467b510c..a4f5e0ef7fd 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2012 Open Information Security Foundation +/* Copyright (C) 2007-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -147,6 +147,8 @@ SCEnumCharMap smtp_decoder_event_table[ ] = { SMTP_DECODER_EVENT_MIME_LONG_HEADER_VALUE }, { "MIME_LONG_BOUNDARY", SMTP_DECODER_EVENT_MIME_BOUNDARY_TOO_LONG }, + { "MIME_LONG_FILENAME", + SMTP_DECODER_EVENT_MIME_LONG_FILENAME }, /* Invalid behavior or content */ { "DUPLICATE_FIELDS", @@ -558,7 +560,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len, * * \param state The smtp state. * - * \retval 0 On suceess. + * \retval 0 On success. * \retval -1 Either when we don't have any new lines to supply anymore or * on failure. */ @@ -873,6 +875,9 @@ static void SetMimeEvents(SMTPState *state) if (msg->anomaly_flags & ANOM_LONG_BOUNDARY) { SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_BOUNDARY_TOO_LONG); } + if (msg->anomaly_flags & ANOM_LONG_FILENAME) { + SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_FILENAME); + } } /** @@ -885,7 +890,7 @@ static int SMTPProcessCommandDATA(SMTPState *state, Flow *f, SCEnter(); if (!(state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) { - /* looks like are still waiting for a confirmination from the server */ + /* looks like are still waiting for a confirmation from the server */ return 0; } @@ -3049,7 +3054,7 @@ static int SMTPParserTest03(void) } /* - * \test Test smtp with just delimter instead of . + * \test Test smtp with just delimiter instead of . */ static int SMTPParserTest04(void) { diff --git a/src/app-layer-smtp.h b/src/app-layer-smtp.h index 59c9d5b9f07..66b28b5e834 100644 --- a/src/app-layer-smtp.h +++ b/src/app-layer-smtp.h @@ -50,6 +50,7 @@ enum { SMTP_DECODER_EVENT_MIME_LONG_HEADER_NAME, SMTP_DECODER_EVENT_MIME_LONG_HEADER_VALUE, SMTP_DECODER_EVENT_MIME_BOUNDARY_TOO_LONG, + SMTP_DECODER_EVENT_MIME_LONG_FILENAME, /* Invalid behavior or content */ SMTP_DECODER_EVENT_DUPLICATE_FIELDS, diff --git a/src/decode-erspan.c b/src/decode-erspan.c index d95c93b5047..dd2515bb009 100644 --- a/src/decode-erspan.c +++ b/src/decode-erspan.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015 Open Information Security Foundation +/* Copyright (C) 2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -27,7 +27,7 @@ * * \author Victor Julien * - * Decodes ERSPAN + * Decodes ERSPAN Types I and II */ #include "suricata-common.h" @@ -40,10 +40,38 @@ #include "util-debug.h" /** - * \brief Function to decode ERSPAN packets + * \brief Functions to decode ERSPAN Type I and II packets */ -int DecodeERSPAN(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *pkt, uint32_t len, PacketQueue *pq) +bool g_erspan_typeI_enabled = false; + +void DecodeERSPANConfig(void) +{ + int enabled = 0; + if (ConfGetBool("decoder.erspan.typeI.enabled", &enabled) == 1) { + g_erspan_typeI_enabled = (enabled == 1); + } + SCLogDebug("ERSPAN Type I decode support %s", g_erspan_typeI_enabled ? "enabled" : "disabled"); +} + +/** + * \brief ERSPAN Type I + */ +int DecodeERSPANTypeI(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, + const uint8_t *pkt, uint32_t len, PacketQueue *pq) +{ + if (unlikely(!g_erspan_typeI_enabled)) + return TM_ECODE_FAILED; + + StatsIncr(tv, dtv->counter_erspan); + + return DecodeEthernet(tv, dtv, p, pkt, len, pq); +} + +/** + * \brief ERSPAN Type II + */ +int DecodeERSPANTypeII(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *pkt, uint32_t len, PacketQueue *pq) { StatsIncr(tv, dtv->counter_erspan); diff --git a/src/decode-erspan.h b/src/decode-erspan.h index 2f81d1e4a38..5b4af04ea89 100644 --- a/src/decode-erspan.h +++ b/src/decode-erspan.h @@ -34,4 +34,5 @@ typedef struct ErspanHdr_ { uint32_t padding; } __attribute__((__packed__)) ErspanHdr; +void DecodeERSPANConfig(void); #endif /* __DECODE_ERSPAN_H__ */ diff --git a/src/decode-gre.c b/src/decode-gre.c index 969a888243a..01e2553edc6 100644 --- a/src/decode-gre.c +++ b/src/decode-gre.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2013 Open Information Security Foundation +/* Copyright (C) 2007-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -66,7 +66,7 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p { case GRE_VERSION_0: - /* GRE version 0 doenst support the fields below RFC 1701 */ + /* GRE version 0 doesn't support the fields below RFC 1701 */ /** * \todo We need to make sure this does not allow bypassing @@ -130,7 +130,7 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p case GRE_VERSION_1: - /* GRE version 1 doenst support the fields below RFC 1701 */ + /* GRE version 1 doesn't support the fields below RFC 1701 */ /** * \todo We need to make sure this does not allow bypassing @@ -252,8 +252,16 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p case ETHERNET_TYPE_ERSPAN: { if (pq != NULL) { + // Determine if it's Type I or Type II based on the flags in the GRE header. + // Type I: 0|0|0|0|0|00000|000000000|00000 + // Type II: 0|0|0|1|0|00000|000000000|00000 + // Seq Packet *tp = PacketTunnelPktSetup(tv, dtv, p, pkt + header_len, - len - header_len, DECODE_TUNNEL_ERSPAN, pq); + len - header_len, + GRE_FLAG_ISSET_SQ(p->greh) == 0 ? + DECODE_TUNNEL_ERSPANI : + DECODE_TUNNEL_ERSPANII, + pq); if (tp != NULL) { PKT_SET_SRC(tp, PKT_SRC_DECODER_GRE); PacketEnqueue(pq,tp); diff --git a/src/decode.c b/src/decode.c index a9f32fda445..67c5a49aa3e 100644 --- a/src/decode.c +++ b/src/decode.c @@ -28,7 +28,7 @@ * example we have DecodeIPV4() for IPv4 and DecodePPP() for * PPP. * - * These functions have all a pkt and and a len argument which + * These functions have all a pkt and a len argument which * are respectively a pointer to the protocol data and the length * of this protocol data. * @@ -68,8 +68,9 @@ #include "output-flow.h" #include "flow-storage.h" +uint32_t default_packet_size = 0; extern bool stats_decoder_events; -const char *stats_decoder_events_prefix; +extern const char *stats_decoder_events_prefix; extern bool stats_stream_events; int DecodeTunnel(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, @@ -87,8 +88,10 @@ int DecodeTunnel(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, return DecodeVLAN(tv, dtv, p, pkt, len, pq); case DECODE_TUNNEL_ETHERNET: return DecodeEthernet(tv, dtv, p, pkt, len, pq); - case DECODE_TUNNEL_ERSPAN: - return DecodeERSPAN(tv, dtv, p, pkt, len, pq); + case DECODE_TUNNEL_ERSPANII: + return DecodeERSPANTypeII(tv, dtv, p, pkt, len, pq); + case DECODE_TUNNEL_ERSPANI: + return DecodeERSPANTypeI(tv, dtv, p, pkt, len, pq); default: SCLogDebug("FIXME: DecodeTunnel: protocol %" PRIu32 " not supported.", proto); break; @@ -109,7 +112,7 @@ void PacketFree(Packet *p) * \brief Finalize decoding of a packet * * This function needs to be call at the end of decode - * functions when decoding has been succesful. + * functions when decoding has been successful. * */ void PacketDecodeFinalize(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p) @@ -286,7 +289,7 @@ Packet *PacketTunnelPktSetup(ThreadVars *tv, DecodeThreadVars *dtv, Packet *pare SCReturnPtr(NULL, "Packet"); } - /* copy packet and set lenght, proto */ + /* copy packet and set length, proto */ PacketCopyData(p, pkt, len); p->recursion_level = parent->recursion_level + 1; p->ts.tv_sec = parent->ts.tv_sec; @@ -376,13 +379,14 @@ Packet *PacketDefragPktSetup(Packet *parent, const uint8_t *pkt, uint32_t len, u p->vlan_id[0] = parent->vlan_id[0]; p->vlan_id[1] = parent->vlan_id[1]; p->vlan_idx = parent->vlan_idx; + p->livedev = parent->livedev; SCReturnPtr(p, "Packet"); } /** * \brief inform defrag "parent" that a pseudo packet is - * now assosiated to it. + * now associated to it. */ void PacketDefragPktSetupParent(Packet *parent) { @@ -644,7 +648,7 @@ void DecodeThreadVarsFree(ThreadVars *tv, DecodeThreadVars *dtv) } /** - * \brief Set data for Packet and set length when zeo copy is used + * \brief Set data for Packet and set length when zero copy is used * * \param Pointer to the Packet to modify * \param Pointer to the data @@ -728,6 +732,7 @@ void DecodeGlobalConfig(void) { DecodeTeredoConfig(); DecodeVXLANConfig(); + DecodeERSPANConfig(); } /** diff --git a/src/decode.h b/src/decode.h index c31b83792a1..1c16114c760 100644 --- a/src/decode.h +++ b/src/decode.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2013 Open Information Security Foundation +/* Copyright (C) 2007-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -161,7 +161,7 @@ typedef struct Address_ { (a)->addr_data32[3] = 0; \ } while (0) -/* Set the IPv6 addressesinto the Addrs of the Packet. +/* Set the IPv6 addresses into the Addrs of the Packet. * Make sure p->ip6h is initialized and validated. */ #define SET_IPV6_SRC_ADDR(p, a) do { \ (a)->family = AF_INET6; \ @@ -386,7 +386,7 @@ typedef struct PktProfiling_ { #endif /* PROFILING */ -/* forward declartion since Packet struct definition requires this */ +/* forward declaration since Packet struct definition requires this */ struct PacketQueue_; /* sizes of the members: @@ -615,7 +615,7 @@ extern int g_default_mtu; #define DEFAULT_PACKET_SIZE (DEFAULT_MTU + ETHERNET_HEADER_LEN) /* storage: maximum ip packet size + link header */ #define MAX_PAYLOAD_SIZE (IPV6_HEADER_LEN + 65536 + 28) -uint32_t default_packet_size; +extern uint32_t default_packet_size; #define SIZE_OF_PACKET (default_packet_size + sizeof(Packet)) typedef struct PacketQueue_ { @@ -889,7 +889,8 @@ void CaptureStatsSetup(ThreadVars *tv, CaptureStats *s); enum DecodeTunnelProto { DECODE_TUNNEL_ETHERNET, - DECODE_TUNNEL_ERSPAN, + DECODE_TUNNEL_ERSPANII, + DECODE_TUNNEL_ERSPANI, DECODE_TUNNEL_VLAN, DECODE_TUNNEL_IPV4, DECODE_TUNNEL_IPV6, @@ -943,6 +944,8 @@ int DecodeVLAN(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint int DecodeVXLAN(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint32_t, PacketQueue *); int DecodeMPLS(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint32_t, PacketQueue *); int DecodeERSPAN(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint32_t, PacketQueue *); +int DecodeERSPANTypeII(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint32_t, PacketQueue *); +int DecodeERSPANTypeI(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint32_t, PacketQueue *); int DecodeTEMPLATE(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint32_t, PacketQueue *); #ifdef UNITTESTS @@ -1089,7 +1092,7 @@ void DecodeUnregisterCounters(void); #define PKT_ALLOC (1<<3) /**< Packet was alloc'd this run, needs to be freed */ #define PKT_HAS_TAG (1<<4) /**< Packet has matched a tag */ #define PKT_STREAM_ADD (1<<5) /**< Packet payload was added to reassembled stream */ -#define PKT_STREAM_EST (1<<6) /**< Packet is part of establised stream */ +#define PKT_STREAM_EST (1<<6) /**< Packet is part of established stream */ #define PKT_STREAM_EOF (1<<7) /**< Stream is in eof state */ #define PKT_HAS_FLOW (1<<8) #define PKT_PSEUDO_STREAM_END (1<<9) /**< Pseudo packet to end the stream */ diff --git a/src/defrag-hash.c b/src/defrag-hash.c index bec87f9d39b..6eee4ea9618 100644 --- a/src/defrag-hash.c +++ b/src/defrag-hash.c @@ -25,6 +25,13 @@ #include "util-misc.h" #include "util-hash-lookup3.h" +/** defrag tracker hash table */ +DefragTrackerHashRow *defragtracker_hash; +DefragConfig defrag_config; +SC_ATOMIC_DECLARE(uint64_t,defrag_memuse); +SC_ATOMIC_DECLARE(unsigned int,defragtracker_counter); +SC_ATOMIC_DECLARE(unsigned int,defragtracker_prune_idx); + static DefragTracker *DefragTrackerGetUsedDefragTracker(void); /** queue with spare tracker */ diff --git a/src/defrag-hash.h b/src/defrag-hash.h index f64208bfaea..2716a6c9010 100644 --- a/src/defrag-hash.h +++ b/src/defrag-hash.h @@ -62,7 +62,7 @@ typedef struct DefragTrackerHashRow_ { } DefragTrackerHashRow; /** defrag tracker hash table */ -DefragTrackerHashRow *defragtracker_hash; +extern DefragTrackerHashRow *defragtracker_hash; #define DEFRAG_VERBOSE 0 #define DEFRAG_QUIET 1 @@ -84,10 +84,10 @@ typedef struct DefragConfig_ { #define DEFRAG_CHECK_MEMCAP(size) \ ((((uint64_t)SC_ATOMIC_GET(defrag_memuse) + (uint64_t)(size)) <= SC_ATOMIC_GET(defrag_config.memcap))) -DefragConfig defrag_config; -SC_ATOMIC_DECLARE(uint64_t,defrag_memuse); -SC_ATOMIC_DECLARE(unsigned int,defragtracker_counter); -SC_ATOMIC_DECLARE(unsigned int,defragtracker_prune_idx); +extern DefragConfig defrag_config; +SC_ATOMIC_EXTERN(uint64_t,defrag_memuse); +SC_ATOMIC_EXTERN(unsigned int,defragtracker_counter); +SC_ATOMIC_EXTERN(unsigned int,defragtracker_prune_idx); void DefragInitConfig(char quiet); void DefragHashShutdown(void); diff --git a/src/detect-engine-address.c b/src/detect-engine-address.c index dc7803afa6f..62b881f2ae8 100644 --- a/src/detect-engine-address.c +++ b/src/detect-engine-address.c @@ -529,9 +529,6 @@ int DetectAddressParseString(DetectAddress *dd, const char *str) int r = 0; char ipstr[256]; - while (*str != '\0' && *str == ' ') - str++; - /* shouldn't see 'any' here */ BUG_ON(strcasecmp(str, "any") == 0); @@ -738,6 +735,9 @@ static int DetectAddressSetup(DetectAddressHead *gh, const char *s) { SCLogDebug("gh %p, s %s", gh, s); + while (*s != '\0' && isspace(*s)) + s++; + if (strcasecmp(s, "any") == 0) { SCLogDebug("adding 0.0.0.0/0 and ::/0 as we\'re handling \'any\'"); diff --git a/src/detect-filestore.c b/src/detect-filestore.c index c2d1340c22a..d3b516c013f 100644 --- a/src/detect-filestore.c +++ b/src/detect-filestore.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2012 Open Information Security Foundation +/* Copyright (C) 2007-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -344,6 +344,9 @@ static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, const ch sm->type = DETECT_FILESTORE; if (str != NULL && strlen(str) > 0) { + char str_0[32]; + char str_1[32]; + char str_2[32]; SCLogDebug("str %s", str); ret = pcre_exec(parse_regex, parse_regex_study, str, strlen(str), 0, 0, ov, MAX_SUBSTRINGS); @@ -353,29 +356,28 @@ static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, const ch } if (ret > 1) { - const char *str_ptr; - res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 1, &str_ptr); + res = pcre_copy_substring((char *)str, ov, MAX_SUBSTRINGS, 1, str_0, sizeof(str_0)); if (res < 0) { - SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed"); goto error; } - args[0] = (char *)str_ptr; + args[0] = (char *)str_0; if (ret > 2) { - res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 2, &str_ptr); + res = pcre_copy_substring((char *)str, ov, MAX_SUBSTRINGS, 2, str_1, sizeof(str_1)); if (res < 0) { - SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed"); goto error; } - args[1] = (char *)str_ptr; + args[1] = (char *)str_1; } if (ret > 3) { - res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 3, &str_ptr); + res = pcre_copy_substring((char *)str, ov, MAX_SUBSTRINGS, 3, str_2, sizeof(str_2)); if (res < 0) { - SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed"); goto error; } - args[2] = (char *)str_ptr; + args[2] = (char *)str_2; } } diff --git a/src/detect-flowvar.c b/src/detect-flowvar.c index 18850cc39fd..532e8f82005 100644 --- a/src/detect-flowvar.c +++ b/src/detect-flowvar.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2014 Open Information Security Foundation +/* Copyright (C) 2007-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -118,11 +118,10 @@ static int DetectFlowvarSetup (DetectEngineCtx *de_ctx, Signature *s, const char { DetectFlowvarData *fd = NULL; SigMatch *sm = NULL; - char *varname = NULL, *varcontent = NULL; + char varname[64], varcontent[64]; #define MAX_SUBSTRINGS 30 int ret = 0, res = 0; int ov[MAX_SUBSTRINGS]; - const char *str_ptr; uint8_t *content = NULL; uint16_t contentlen = 0; uint32_t contentflags = s->init_data->negated ? DETECT_CONTENT_NEGATED : 0; @@ -133,29 +132,28 @@ static int DetectFlowvarSetup (DetectEngineCtx *de_ctx, Signature *s, const char return -1; } - res = pcre_get_substring((char *)rawstr, ov, MAX_SUBSTRINGS, 1, &str_ptr); + res = pcre_copy_substring((char *)rawstr, ov, MAX_SUBSTRINGS, 1, varname, sizeof(varname)); if (res < 0) { - SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed"); return -1; } - varname = (char *)str_ptr; - res = pcre_get_substring((char *)rawstr, ov, MAX_SUBSTRINGS, 2, &str_ptr); + res = pcre_copy_substring((char *)rawstr, ov, MAX_SUBSTRINGS, 2, varcontent, sizeof(varcontent)); if (res < 0) { - SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed"); return -1; } - varcontent = (char *)str_ptr; + int varcontent_index = 0; if (strlen(varcontent) >= 2) { if (varcontent[0] == '"') - varcontent++; + varcontent_index++; if (varcontent[strlen(varcontent)-1] == '"') varcontent[strlen(varcontent)-1] = '\0'; } - SCLogDebug("varcontent %s", varcontent); + SCLogDebug("varcontent %s", &varcontent[varcontent_index]); - res = DetectContentDataParse("flowvar", varcontent, &content, &contentlen); + res = DetectContentDataParse("flowvar", &varcontent[varcontent_index], &content, &contentlen); if (res == -1) goto error; diff --git a/src/detect-parse.c b/src/detect-parse.c index 544b204ac26..d1a9a61a2cf 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -69,6 +69,9 @@ #include "detect-engine-iponly.h" #include "app-layer-detect-proto.h" +/* Table with all SigMatch registrations */ +SigTableElmt sigmatch_table[DETECT_TBLSIZE]; + extern int sc_set_caps; static void SigMatchTransferSigMatchAcrossLists(SigMatch *sm, diff --git a/src/detect-ssl-state.c b/src/detect-ssl-state.c index 01a1cac6e2e..0c5979bc149 100644 --- a/src/detect-ssl-state.c +++ b/src/detect-ssl-state.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2016 Open Information Security Foundation +/* Copyright (C) 2007-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -156,7 +156,7 @@ static int DetectSslStateMatch(DetectEngineThreadCtx *det_ctx, * * \param arg Pointer to the string to be parsed. * - * \retval ssd Pointer to DetectSslStateData on succese. + * \retval ssd Pointer to DetectSslStateData on success. * \retval NULL On failure. */ static DetectSslStateData *DetectSslStateParse(const char *arg) @@ -165,8 +165,8 @@ static DetectSslStateData *DetectSslStateParse(const char *arg) int ret = 0, res = 0; int ov1[MAX_SUBSTRINGS]; int ov2[MAX_SUBSTRINGS]; - const char *str1; - const char *str2; + char str1[64]; + char str2[64]; int negate = 0; uint32_t flags = 0, mask = 0; DetectSslStateData *ssd = NULL; @@ -179,17 +179,16 @@ static DetectSslStateData *DetectSslStateParse(const char *arg) goto error; } - res = pcre_get_substring((char *)arg, ov1, MAX_SUBSTRINGS, 1, &str1); + res = pcre_copy_substring((char *)arg, ov1, MAX_SUBSTRINGS, 1, str1, sizeof(str1)); if (res < 0) { - SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed"); goto error; } negate = !strcmp("!", str1); - pcre_free_substring(str1); - res = pcre_get_substring((char *)arg, ov1, MAX_SUBSTRINGS, 2, &str1); + res = pcre_copy_substring((char *)arg, ov1, MAX_SUBSTRINGS, 2, str1, sizeof(str1)); if (res < 0) { - SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed"); goto error; } @@ -219,11 +218,9 @@ static DetectSslStateData *DetectSslStateParse(const char *arg) goto error; } - pcre_free_substring(str1); - - res = pcre_get_substring((char *)arg, ov1, MAX_SUBSTRINGS, 3, &str1); + res = pcre_copy_substring((char *)arg, ov1, MAX_SUBSTRINGS, 3, str1, sizeof(str1)); if (res < 0) { - SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed"); goto error; } while (res > 0) { @@ -235,17 +232,16 @@ static DetectSslStateData *DetectSslStateParse(const char *arg) goto error; } - res = pcre_get_substring((char *)str1, ov2, MAX_SUBSTRINGS, 1, &str2); + res = pcre_copy_substring((char *)str1, ov2, MAX_SUBSTRINGS, 1, str2, sizeof(str2)); if (res < 0) { - SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed"); goto error; } negate = !strcmp("!", str2); - pcre_free_substring(str2); - res = pcre_get_substring((char *)str1, ov2, MAX_SUBSTRINGS, 2, &str2); + res = pcre_copy_substring((char *)str1, ov2, MAX_SUBSTRINGS, 2, str2, sizeof(str2)); if (res <= 0) { - SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed"); goto error; } if (strcmp("client_hello", str2) == 0) { @@ -274,16 +270,14 @@ static DetectSslStateData *DetectSslStateParse(const char *arg) goto error; } - res = pcre_get_substring((char *)str1, ov2, MAX_SUBSTRINGS, 3, &str2); + res = pcre_copy_substring((char *)str1, ov2, MAX_SUBSTRINGS, 3, str2, sizeof(str2)); if (res < 0) { - SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed"); goto error; } - pcre_free_substring(str1); - str1 = str2; + memcpy(str1, str2, sizeof(str1)); } - pcre_free_substring(str1); if ( (ssd = SCMalloc(sizeof(DetectSslStateData))) == NULL) { goto error; diff --git a/src/detect.h b/src/detect.h index 0a895a8990e..6951f457996 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1445,7 +1445,7 @@ typedef struct DetectEngineMasterCtx_ { } DetectEngineMasterCtx; /* Table with all SigMatch registrations */ -SigTableElmt sigmatch_table[DETECT_TBLSIZE]; +extern SigTableElmt sigmatch_table[DETECT_TBLSIZE]; /** Remember to add the options in SignatureIsIPOnly() at detect.c otherwise it wont be part of a signature group */ diff --git a/src/flow-hash.c b/src/flow-hash.c index aa434ec69f0..26aa7efd747 100644 --- a/src/flow-hash.c +++ b/src/flow-hash.c @@ -49,6 +49,7 @@ #define FLOW_DEFAULT_FLOW_PRUNE 5 +FlowBucket *flow_hash; SC_ATOMIC_EXTERN(unsigned int, flow_prune_idx); SC_ATOMIC_EXTERN(unsigned int, flow_flags); diff --git a/src/flow-manager.c b/src/flow-manager.c index e515332192c..e7f683ed986 100644 --- a/src/flow-manager.c +++ b/src/flow-manager.c @@ -71,6 +71,9 @@ /* Run mode selected at suricata.c */ extern int run_mode; +/** queue to pass flows to cleanup/log thread(s) */ +FlowQueue flow_recycle_q; + /* multi flow mananger support */ static uint32_t flowmgr_number = 1; /* atomic counter for flow managers, to assign instance id */ @@ -83,6 +86,11 @@ SC_ATOMIC_DECLARE(uint32_t, flowrec_cnt); SC_ATOMIC_EXTERN(unsigned int, flow_flags); +SCCtrlCondT flow_manager_ctrl_cond; +SCCtrlMutex flow_manager_ctrl_mutex; + +SCCtrlCondT flow_recycler_ctrl_cond; +SCCtrlMutex flow_recycler_ctrl_mutex; typedef FlowProtoTimeout *FlowProtoTimeoutPtr; SC_ATOMIC_DECLARE(FlowProtoTimeoutPtr, flow_timeouts); diff --git a/src/flow-manager.h b/src/flow-manager.h index 117c98df6d0..8ce882c5912 100644 --- a/src/flow-manager.h +++ b/src/flow-manager.h @@ -29,8 +29,8 @@ void FlowTimeoutsInit(void); void FlowTimeoutsEmergency(void); /** flow manager scheduling condition */ -SCCtrlCondT flow_manager_ctrl_cond; -SCCtrlMutex flow_manager_ctrl_mutex; +extern SCCtrlCondT flow_manager_ctrl_cond; +extern SCCtrlMutex flow_manager_ctrl_mutex; #define FlowWakeupFlowManagerThread() SCCtrlCondSignal(&flow_manager_ctrl_cond) void FlowManagerThreadSpawn(void); @@ -38,8 +38,8 @@ void FlowDisableFlowManagerThread(void); void FlowMgrRegisterTests (void); /** flow recycler scheduling condition */ -SCCtrlCondT flow_recycler_ctrl_cond; -SCCtrlMutex flow_recycler_ctrl_mutex; +extern SCCtrlCondT flow_recycler_ctrl_cond; +extern SCCtrlMutex flow_recycler_ctrl_mutex; #define FlowWakeupFlowRecyclerThread() \ SCCtrlCondSignal(&flow_recycler_ctrl_cond) diff --git a/src/flow-private.h b/src/flow-private.h index 2e712367ec9..45315eda9b3 100644 --- a/src/flow-private.h +++ b/src/flow-private.h @@ -83,21 +83,21 @@ enum { /** FlowProto specific timeouts and free/state functions */ -FlowProtoTimeout flow_timeouts_normal[FLOW_PROTO_MAX]; -FlowProtoTimeout flow_timeouts_emerg[FLOW_PROTO_MAX]; -FlowProtoFreeFunc flow_freefuncs[FLOW_PROTO_MAX]; +extern FlowProtoTimeout flow_timeouts_normal[FLOW_PROTO_MAX]; +extern FlowProtoTimeout flow_timeouts_emerg[FLOW_PROTO_MAX]; +extern FlowProtoFreeFunc flow_freefuncs[FLOW_PROTO_MAX]; /** spare/unused/prealloced flows live here */ -FlowQueue flow_spare_q; +extern FlowQueue flow_spare_q; /** queue to pass flows to cleanup/log thread(s) */ -FlowQueue flow_recycle_q; +extern FlowQueue flow_recycle_q; -FlowBucket *flow_hash; -FlowConfig flow_config; +extern FlowBucket *flow_hash; +extern FlowConfig flow_config; /** flow memuse counter (atomic), for enforcing memcap limit */ -SC_ATOMIC_DECLARE(uint64_t, flow_memuse); +SC_ATOMIC_EXTERN(uint64_t, flow_memuse); #endif /* __FLOW_PRIVATE_H__ */ diff --git a/src/flow.c b/src/flow.c index 5fcf5313574..0d3a187ef4e 100644 --- a/src/flow.c +++ b/src/flow.c @@ -83,6 +83,20 @@ SC_ATOMIC_DECLARE(unsigned int, flow_prune_idx); /** atomic flags */ SC_ATOMIC_DECLARE(unsigned int, flow_flags); +/** FlowProto specific timeouts and free/state functions */ + +FlowProtoTimeout flow_timeouts_normal[FLOW_PROTO_MAX]; +FlowProtoTimeout flow_timeouts_emerg[FLOW_PROTO_MAX]; +FlowProtoFreeFunc flow_freefuncs[FLOW_PROTO_MAX]; + +/** spare/unused/prealloced flows live here */ +FlowQueue flow_spare_q; + +FlowConfig flow_config; + +/** flow memuse counter (atomic), for enforcing memcap limit */ +SC_ATOMIC_DECLARE(uint64_t, flow_memuse); + void FlowRegisterTests(void); void FlowInitFlowProto(void); int FlowSetProtoFreeFunc(uint8_t, void (*Free)(void *)); @@ -1113,6 +1127,19 @@ void FlowUpdateState(Flow *f, enum FlowState s) } } +/** + * \brief Get flow last time as individual values. + * + * Instead of returning a pointer to the timeval copy the timeval + * parts into output pointers to make it simpler to call from Rust + * over FFI using only basic data types. + */ +void FlowGetLastTimeAsParts(Flow *flow, uint64_t *secs, uint64_t *usecs) +{ + *secs = (uint64_t)flow->lastts.tv_sec; + *usecs = (uint64_t)flow->lastts.tv_usec; +} + /************************************Unittests*******************************/ #ifdef UNITTESTS diff --git a/src/flow.h b/src/flow.h index cf6e729581c..b2eb2179648 100644 --- a/src/flow.h +++ b/src/flow.h @@ -537,6 +537,8 @@ uint64_t FlowGetMemuse(void); int GetFlowBypassInfoID(void); void RegisterFlowBypassInfo(void); +void FlowGetLastTimeAsParts(Flow *flow, uint64_t *secs, uint64_t *usecs); + /** ----- Inline functions ----- */ /** \brief Set the No Packet Inspection Flag without locking the flow. diff --git a/src/host.c b/src/host.c index 548b31abbbc..4e79be14daf 100644 --- a/src/host.c +++ b/src/host.c @@ -45,8 +45,15 @@ static Host *HostGetUsedHost(void); +/** host hash table */ +HostHashRow *host_hash; /** queue with spare hosts */ static HostQueue host_spare_q; +HostConfig host_config; + +SC_ATOMIC_DECLARE(uint64_t,host_memuse); +SC_ATOMIC_DECLARE(uint32_t,host_counter); +SC_ATOMIC_DECLARE(uint32_t,host_prune_idx); /** size of the host object. Maybe updated in HostInitConfig to include * the storage APIs additions. */ diff --git a/src/host.h b/src/host.h index 9a7b811b2a8..e44cc5368a2 100644 --- a/src/host.h +++ b/src/host.h @@ -87,7 +87,7 @@ typedef struct HostHashRow_ { } __attribute__((aligned(CLS))) HostHashRow; /** host hash table */ -HostHashRow *host_hash; +extern HostHashRow *host_hash; #define HOST_VERBOSE 0 #define HOST_QUIET 1 @@ -128,10 +128,10 @@ typedef struct HostConfig_ { } \ } while (0) -HostConfig host_config; -SC_ATOMIC_DECLARE(uint64_t,host_memuse); -SC_ATOMIC_DECLARE(uint32_t,host_counter); -SC_ATOMIC_DECLARE(uint32_t,host_prune_idx); +extern HostConfig host_config; +SC_ATOMIC_EXTERN(uint64_t,host_memuse); +SC_ATOMIC_EXTERN(uint32_t,host_counter); +SC_ATOMIC_EXTERN(uint32_t,host_prune_idx); void HostInitConfig(char quiet); void HostShutdown(void); diff --git a/src/ippair.c b/src/ippair.c index 1d7c899082d..07d4e4efb98 100644 --- a/src/ippair.c +++ b/src/ippair.c @@ -44,8 +44,14 @@ static IPPair *IPPairGetUsedIPPair(void); +/** ippair hash table */ +IPPairHashRow *ippair_hash; /** queue with spare ippairs */ static IPPairQueue ippair_spare_q; +IPPairConfig ippair_config; +SC_ATOMIC_DECLARE(uint64_t,ippair_memuse); +SC_ATOMIC_DECLARE(uint32_t,ippair_counter); +SC_ATOMIC_DECLARE(uint32_t,ippair_prune_idx); /** size of the ippair object. Maybe updated in IPPairInitConfig to include * the storage APIs additions. */ diff --git a/src/ippair.h b/src/ippair.h index e5142856ed1..d89de6752c4 100644 --- a/src/ippair.h +++ b/src/ippair.h @@ -84,7 +84,7 @@ typedef struct IPPairHashRow_ { } __attribute__((aligned(CLS))) IPPairHashRow; /** ippair hash table */ -IPPairHashRow *ippair_hash; +extern IPPairHashRow *ippair_hash; #define IPPAIR_VERBOSE 0 #define IPPAIR_QUIET 1 @@ -125,10 +125,10 @@ typedef struct IPPairConfig_ { } \ } while (0) -IPPairConfig ippair_config; -SC_ATOMIC_DECLARE(uint64_t,ippair_memuse); -SC_ATOMIC_DECLARE(uint32_t,ippair_counter); -SC_ATOMIC_DECLARE(uint32_t,ippair_prune_idx); +extern IPPairConfig ippair_config; +SC_ATOMIC_EXTERN(uint64_t,ippair_memuse); +SC_ATOMIC_EXTERN(uint32_t,ippair_counter); +SC_ATOMIC_EXTERN(uint32_t,ippair_prune_idx); void IPPairInitConfig(char quiet); void IPPairShutdown(void); diff --git a/src/output-json-stats.c b/src/output-json-stats.c index 188a3311a9e..cc5a59b57e1 100644 --- a/src/output-json-stats.c +++ b/src/output-json-stats.c @@ -51,7 +51,7 @@ #define MODULE_NAME "JsonStatsLog" extern bool stats_decoder_events; -const char *stats_decoder_events_prefix; +extern const char *stats_decoder_events_prefix; /** * specify which engine info will be printed in stats log. diff --git a/src/output-json-tls.c b/src/output-json-tls.c index fb4685f6069..2bd08c8c089 100644 --- a/src/output-json-tls.c +++ b/src/output-json-tls.c @@ -51,7 +51,7 @@ #include "output-json.h" #include "output-json-tls.h" -SC_ATOMIC_DECLARE(unsigned int, cert_id); +SC_ATOMIC_EXTERN(unsigned int, cert_id); #define MODULE_NAME "LogTlsLog" #define DEFAULT_LOG_FILENAME "tls.json" diff --git a/src/runmodes.c b/src/runmodes.c index 47f774493d8..c693f0ec582 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -56,6 +56,7 @@ #include "counters.h" int debuglog_enabled = 0; +int threading_set_cpu_affinity = FALSE; /* Runmode Global Thread Names */ const char *thread_name_autofp = "RX"; diff --git a/src/runmodes.h b/src/runmodes.h index 451ab2248a7..c1202b75c18 100644 --- a/src/runmodes.h +++ b/src/runmodes.h @@ -109,7 +109,7 @@ int RunModeNeedsBypassManager(void); #include "runmode-netmap.h" #include "runmode-windivert.h" -int threading_set_cpu_affinity; +extern int threading_set_cpu_affinity; extern float threading_detect_ratio; extern int debuglog_enabled; diff --git a/src/source-erf-dag.c b/src/source-erf-dag.c index 9d4e34fc54c..1bd9a506f1c 100644 --- a/src/source-erf-dag.c +++ b/src/source-erf-dag.c @@ -432,16 +432,17 @@ ProcessErfDagRecords(ErfDagThreadVars *ewtn, uint8_t *top, uint32_t *pkts_read) /* Only support ethernet at this time. */ switch (hdr_type & 0x7f) { - case TYPE_PAD: + case ERF_TYPE_PAD: + case ERF_TYPE_META: /* Skip. */ continue; - case TYPE_DSM_COLOR_ETH: - case TYPE_COLOR_ETH: - case TYPE_COLOR_HASH_ETH: + case ERF_TYPE_DSM_COLOR_ETH: + case ERF_TYPE_COLOR_ETH: + case ERF_TYPE_COLOR_HASH_ETH: /* In these types the color value overwrites the lctr * (drop count). */ break; - case TYPE_ETH: + case ERF_TYPE_ETH: if (dr->lctr) { StatsAddUI64(ewtn->tv, ewtn->drops, SCNtohs(dr->lctr)); } diff --git a/src/stream-tcp-list.c b/src/stream-tcp-list.c index 31401d02367..cb47ee8e923 100644 --- a/src/stream-tcp-list.c +++ b/src/stream-tcp-list.c @@ -141,7 +141,7 @@ static inline bool CheckOverlap(struct TCPSEG *tree, TcpSegment *seg) return true; // prev's right edge is beyond our seq, overlap const uint32_t prev_re = SEG_SEQ_RIGHT_EDGE(prev); - if (SEQ_GT(prev_re, prev->seq)) + if (SEQ_GT(prev_re, seg->seq)) return true; } diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c index e48659a225a..baadd91b1d3 100644 --- a/src/stream-tcp-reassemble.c +++ b/src/stream-tcp-reassemble.c @@ -658,7 +658,7 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre } static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream, - Packet *p, enum StreamUpdateDir dir) + Packet *p) { uint8_t flag = 0; @@ -678,20 +678,11 @@ static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream, flag |= STREAM_EOF; } - if (dir == UPDATE_DIR_OPPOSING) { - if (p->flowflags & FLOW_PKT_TOSERVER) { - flag |= STREAM_TOCLIENT; - } else { - flag |= STREAM_TOSERVER; - } + if (&ssn->client == stream) { + flag |= STREAM_TOSERVER; } else { - if (p->flowflags & FLOW_PKT_TOSERVER) { - flag |= STREAM_TOSERVER; - } else { - flag |= STREAM_TOCLIENT; - } + flag |= STREAM_TOCLIENT; } - if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) { flag |= STREAM_DEPTH; } @@ -1029,7 +1020,7 @@ static int ReassembleUpdateAppLayer (ThreadVars *tv, int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, NULL, mydata_len, - StreamGetAppLayerFlags(ssn, *stream, p, dir)|STREAM_GAP); + StreamGetAppLayerFlags(ssn, *stream, p)|STREAM_GAP); AppLayerProfilingStore(ra_ctx->app_tctx, p); StreamTcpSetEvent(p, STREAM_REASSEMBLY_SEQ_GAP); @@ -1087,7 +1078,7 @@ static int ReassembleUpdateAppLayer (ThreadVars *tv, /* update the app-layer */ (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, (uint8_t *)mydata, mydata_len, - StreamGetAppLayerFlags(ssn, *stream, p, dir)); + StreamGetAppLayerFlags(ssn, *stream, p)); AppLayerProfilingStore(ra_ctx->app_tctx, p); SCReturnInt(0); @@ -1134,7 +1125,7 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, /* send EOF to app layer */ AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream, NULL, 0, - StreamGetAppLayerFlags(ssn, stream, p, dir)); + StreamGetAppLayerFlags(ssn, stream, p)); AppLayerProfilingStore(ra_ctx->app_tctx, p); SCReturnInt(0); diff --git a/src/stream-tcp.c b/src/stream-tcp.c index df392376ba7..a99ce68b65b 100644 --- a/src/stream-tcp.c +++ b/src/stream-tcp.c @@ -116,6 +116,7 @@ static SCMutex ssn_pool_mutex = SCMUTEX_INITIALIZER; /**< init only, protect ini static uint64_t ssn_pool_cnt = 0; /** counts ssns, protected by ssn_pool_mutex */ #endif +TcpStreamCnf stream_config; uint64_t StreamTcpReassembleMemuseGlobalCounter(void); SC_ATOMIC_DECLARE(uint64_t, st_memuse); diff --git a/src/stream-tcp.h b/src/stream-tcp.h index e243eebb5ef..d5d1866417a 100644 --- a/src/stream-tcp.h +++ b/src/stream-tcp.h @@ -103,7 +103,7 @@ typedef struct StreamTcpThread_ { TcpReassemblyThreadCtx *ra_ctx; } StreamTcpThread; -TcpStreamCnf stream_config; +extern TcpStreamCnf stream_config; void StreamTcpInitConfig (char); void StreamTcpFreeConfig(char); void StreamTcpRegisterTests (void); diff --git a/src/suricata-common.h b/src/suricata-common.h index b55509c52eb..ed71a88b8a9 100644 --- a/src/suricata-common.h +++ b/src/suricata-common.h @@ -497,5 +497,9 @@ extern int g_ut_covered; #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0])) +#ifndef NAME_MAX +#define NAME_MAX 255 +#endif + #endif /* __SURICATA_COMMON_H__ */ diff --git a/src/suricata.h b/src/suricata.h index a05febdb523..3fd30650a95 100644 --- a/src/suricata.h +++ b/src/suricata.h @@ -125,10 +125,7 @@ enum { #include "runmodes.h" -/* queue's between various other threads - * XXX move to the TmQueue structure later - */ -PacketQueue trans_q[256]; +extern PacketQueue trans_q[256]; typedef struct SCInstance_ { enum RunModes run_mode; diff --git a/src/tm-modules.c b/src/tm-modules.c index d9478cc8d89..9f0ec6e7df5 100644 --- a/src/tm-modules.c +++ b/src/tm-modules.c @@ -30,6 +30,8 @@ #include "threads.h" #include "util-logopenfile.h" +TmModule tmm_modules[TMM_SIZE]; + void TmModuleDebugList(void) { TmModule *t; diff --git a/src/tm-modules.h b/src/tm-modules.h index 5d41de617b6..9493d730548 100644 --- a/src/tm-modules.h +++ b/src/tm-modules.h @@ -70,7 +70,7 @@ typedef struct TmModule_ { uint8_t flags; } TmModule; -TmModule tmm_modules[TMM_SIZE]; +extern TmModule tmm_modules[TMM_SIZE]; /** * Structure that output modules use to maintain private data. diff --git a/src/tm-queuehandlers.c b/src/tm-queuehandlers.c index 862fc7d396a..6b2a6b98a67 100644 --- a/src/tm-queuehandlers.c +++ b/src/tm-queuehandlers.c @@ -34,6 +34,8 @@ #include "tmqh-packetpool.h" #include "tmqh-flow.h" +Tmqh tmqh_table[TMQH_SIZE]; + void TmqhSetup (void) { memset(&tmqh_table, 0, sizeof(tmqh_table)); diff --git a/src/tm-queuehandlers.h b/src/tm-queuehandlers.h index e4dc329db3d..85a4520be9d 100644 --- a/src/tm-queuehandlers.h +++ b/src/tm-queuehandlers.h @@ -42,7 +42,7 @@ typedef struct Tmqh_ { void (*RegisterTests)(void); } Tmqh; -Tmqh tmqh_table[TMQH_SIZE]; +extern Tmqh tmqh_table[TMQH_SIZE]; void TmqhSetup (void); void TmqhCleanup(void); diff --git a/src/tm-threads.c b/src/tm-threads.c index c7cc6fe5049..aca6d96467d 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -80,6 +80,11 @@ ThreadVars *tv_root[TVT_MAX] = { NULL }; /* lock to protect tv_root */ SCMutex tv_root_lock = SCMUTEX_INITIALIZER; +/* queue's between various other threads + * XXX move to the TmQueue structure later + */ +PacketQueue trans_q[256]; + /** * \brief Check if a thread flag is set. * @@ -172,6 +177,27 @@ TmEcode TmThreadsSlotVarRun(ThreadVars *tv, Packet *p, return TM_ECODE_OK; } +/** \internal + * \brief check 'slot' pre_pq and post_pq at thread cleanup + * and dump detailed info about the state of the packets + * and threads if in a unexpected state. + */ +static void CheckSlot(const TmSlot *slot) +{ + if (slot->slot_pre_pq.len || slot->slot_post_pq.len) { + for (Packet *xp = slot->slot_pre_pq.top; xp != NULL; xp = xp->next) { + SCLogNotice("pre_pq: slot id %u slot tm_id %u pre_pq.len %u packet src %s", + slot->id, slot->tm_id, slot->slot_pre_pq.len, PktSrcToString(xp->pkt_src)); + } + for (Packet *xp = slot->slot_post_pq.top; xp != NULL; xp = xp->next) { + SCLogNotice("post_pq: slot id %u slot tm_id %u post_pq.len %u packet src %s", + slot->id, slot->tm_id, slot->slot_post_pq.len, PktSrcToString(xp->pkt_src)); + } + TmThreadDumpThreads(); + abort(); + } +} + #ifndef AFLFUZZ_PCAP_RUNMODE /** \internal @@ -229,27 +255,6 @@ static int TmThreadTimeoutLoop(ThreadVars *tv, TmSlot *s) return r; } -/** \internal - * \brief check 'slot' pre_pq and post_pq at thread cleanup - * and dump detailed info about the state of the packets - * and threads if in a unexpected state. - */ -static void CheckSlot(const TmSlot *slot) -{ - if (slot->slot_pre_pq.len || slot->slot_post_pq.len) { - for (Packet *xp = slot->slot_pre_pq.top; xp != NULL; xp = xp->next) { - SCLogNotice("pre_pq: slot id %u slot tm_id %u pre_pq.len %u packet src %s", - slot->id, slot->tm_id, slot->slot_pre_pq.len, PktSrcToString(xp->pkt_src)); - } - for (Packet *xp = slot->slot_post_pq.top; xp != NULL; xp = xp->next) { - SCLogNotice("post_pq: slot id %u slot tm_id %u post_pq.len %u packet src %s", - slot->id, slot->tm_id, slot->slot_post_pq.len, PktSrcToString(xp->pkt_src)); - } - TmThreadDumpThreads(); - abort(); - } -} - /* pcap/nfq diff --git a/src/unix-manager.c b/src/unix-manager.c index 68af2207572..6de2ecd1425 100644 --- a/src/unix-manager.c +++ b/src/unix-manager.c @@ -57,6 +57,9 @@ #define SOCKET_FILENAME "suricata-command.socket" #define SOCKET_TARGET SOCKET_PATH SOCKET_FILENAME +SCCtrlCondT unix_manager_ctrl_cond; +SCCtrlMutex unix_manager_ctrl_mutex; + #define MAX_FAILED_RULES 20 typedef struct Command_ { diff --git a/src/unix-manager.h b/src/unix-manager.h index a1dbd0835f7..22f21923be1 100644 --- a/src/unix-manager.h +++ b/src/unix-manager.h @@ -26,8 +26,8 @@ #define UNIX_CMD_TAKE_ARGS 1 -SCCtrlCondT unix_manager_ctrl_cond; -SCCtrlMutex unix_manager_ctrl_mutex; +extern SCCtrlCondT unix_manager_ctrl_cond; +extern SCCtrlMutex unix_manager_ctrl_mutex; int UnixManagerInit(void); void UnixManagerThreadSpawn(int mode); diff --git a/src/util-decode-mime.c b/src/util-decode-mime.c index 7dbc814d378..5c0ae1b2856 100644 --- a/src/util-decode-mime.c +++ b/src/util-decode-mime.c @@ -1,4 +1,5 @@ /* Copyright (C) 2012 BAE Systems + * Copyright (C) 2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -1830,30 +1831,63 @@ static int FindMimeHeader(const uint8_t *buf, uint32_t blen, * \param search_start The start of the search (ie. boundary=\") * \param search_end The end of the search (ie. \") * \param tlen The output length of the token (if found) + * \param max_len The maximum offset in which to search + * \param toolong Set if the field value was truncated to max_len. * * \return A pointer to the token if found, otherwise NULL if not found */ -static uint8_t * FindMimeHeaderToken(MimeDecField *field, const char *search_start, - const char *search_end, uint32_t *tlen) +static uint8_t * FindMimeHeaderTokenRestrict(MimeDecField *field, const char *search_start, + const char *search_end, uint32_t *tlen, uint32_t max_len, bool *toolong) { uint8_t *fptr, *tptr = NULL, *tok = NULL; + if (toolong) + *toolong = false; + SCLogDebug("Looking for token: %s", search_start); /* Check for token definition */ - fptr = FindBuffer(field->value, field->value_len, (const uint8_t *)search_start, strlen(search_start)); + size_t ss_len = strlen(search_start); + fptr = FindBuffer(field->value, field->value_len, (const uint8_t *)search_start, ss_len); if (fptr != NULL) { - fptr += strlen(search_start); /* Start at end of start string */ - tok = GetToken(fptr, field->value_len - (fptr - field->value), search_end, - &tptr, tlen); - if (tok != NULL) { - SCLogDebug("Found mime token"); + fptr += ss_len; /* Start at end of start string */ + uint32_t offset = fptr - field->value; + if (offset > field->value_len) { + return tok; + } + tok = GetToken(fptr, field->value_len - offset, search_end, &tptr, tlen); + if (tok == NULL) { + return tok; + } + SCLogDebug("Found mime token"); + + /* Compare the actual token length against the maximum */ + if (toolong && max_len && *tlen > max_len) { + SCLogDebug("Token length %d exceeds length restriction %d; truncating", *tlen, max_len); + *toolong = true; + *tlen = max_len; } } return tok; } +/** + * \brief Finds a mime header token within the specified field + * + * \param field The current field + * \param search_start The start of the search (ie. boundary=\") + * \param search_end The end of the search (ie. \") + * \param tlen The output length of the token (if found) + * + * \return A pointer to the token if found, otherwise NULL if not found + */ +static uint8_t * FindMimeHeaderToken(MimeDecField *field, const char *search_start, + const char *search_end, uint32_t *tlen) +{ + return FindMimeHeaderTokenRestrict(field, search_start, search_end, tlen, 0, NULL); +} + /** * \brief Processes the current line for mime headers and also does post-processing * when all headers found @@ -1899,7 +1933,8 @@ static int ProcessMimeHeaders(const uint8_t *buf, uint32_t len, /* Check for file attachment in content disposition */ field = MimeDecFindField(entity, CTNT_DISP_STR); if (field != NULL) { - bptr = FindMimeHeaderToken(field, "filename=", TOK_END_STR, &blen); + bool truncated_name = false; + bptr = FindMimeHeaderTokenRestrict(field, "filename=", TOK_END_STR, &blen, NAME_MAX, &truncated_name); if (bptr != NULL) { SCLogDebug("File attachment found in disposition"); entity->ctnt_flags |= CTNT_IS_ATTACHMENT; @@ -1912,6 +1947,11 @@ static int ProcessMimeHeaders(const uint8_t *buf, uint32_t len, } memcpy(entity->filename, bptr, blen); entity->filename_len = blen; + + if (truncated_name) { + state->stack->top->data->anomaly_flags |= ANOM_LONG_FILENAME; + state->msg->anomaly_flags |= ANOM_LONG_FILENAME; + } } } @@ -1941,7 +1981,8 @@ static int ProcessMimeHeaders(const uint8_t *buf, uint32_t len, /* Look for file name (if not already found) */ if (!(entity->ctnt_flags & CTNT_IS_ATTACHMENT)) { - bptr = FindMimeHeaderToken(field, "name=", TOK_END_STR, &blen); + bool truncated_name = false; + bptr = FindMimeHeaderTokenRestrict(field, "name=", TOK_END_STR, &blen, NAME_MAX, &truncated_name); if (bptr != NULL) { SCLogDebug("File attachment found"); entity->ctnt_flags |= CTNT_IS_ATTACHMENT; @@ -1954,6 +1995,11 @@ static int ProcessMimeHeaders(const uint8_t *buf, uint32_t len, } memcpy(entity->filename, bptr, blen); entity->filename_len = blen; + + if (truncated_name) { + state->stack->top->data->anomaly_flags |= ANOM_LONG_FILENAME; + state->msg->anomaly_flags |= ANOM_LONG_FILENAME; + } } } @@ -3035,6 +3081,126 @@ static int MimeIsIpv6HostTest01(void) } #undef TEST +static int MimeDecParseLongFilename01(void) +{ + /* contains 276 character filename -- length restricted to 255 chars */ + char mimemsg[] = "Content-Disposition: attachment; filename=\"" + "12characters12characters12characters12characters" + "12characters12characters12characters12characters" + "12characters12characters12characters12characters" + "12characters12characters12characters12characters" + "12characters12characters12characters12characters" + "12characters12characters12characters.exe\""; + + uint32_t line_count = 0; + + MimeDecGetConfig()->decode_base64 = 1; + MimeDecGetConfig()->decode_quoted_printable = 1; + MimeDecGetConfig()->extract_urls = 1; + + /* Init parser */ + MimeDecParseState *state = MimeDecInitParser(&line_count, + TestDataChunkCallback); + + const char *str = "From: Sender1"; + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); + + str = "To: Recipient1"; + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); + + str = "Content-Type: text/plain"; + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); + + /* Contains 276 character filename */ + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)mimemsg, strlen(mimemsg), 1, state)); + + str = ""; + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); + + str = "A simple message line 1"; + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); + + /* Completed */ + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseComplete(state)); + + MimeDecEntity *msg = state->msg; + FAIL_IF_NOT(msg); + + FAIL_IF_NOT(msg->anomaly_flags & ANOM_LONG_FILENAME); + FAIL_IF_NOT(msg->filename_len == NAME_MAX); + + MimeDecFreeEntity(msg); + + /* De Init parser */ + MimeDecDeInitParser(state); + + PASS; +} + +static int MimeDecParseLongFilename02(void) +{ + /* contains 40 character filename and 500+ characters following filename */ + char mimemsg[] = "Content-Disposition: attachment; filename=\"" + "12characters12characters12characters.exe\"; " + "somejunkasfdasfsafasafdsasdasassdssdsd" + "somejunkasfdasfsafasafdsasdasassdssdsd" + "somejunkasfdasfsafasafdsasdasassdssdsd" + "somejunkasfdasfsafasafdsasdasassdssdsd" + "somejunkasfdasfsafasafdsasdasassdssdsd" + "somejunkasfdasfsafasafdsasdasassdssdsd" + "somejunkasfdasfsafasafdsasdasassdssdsd" + "somejunkasfdasfsafasafdsasdasassdssdsd" + "somejunkasfdasfsafasafdsasdasassdssdsd" + "somejunkasfdasfsafasafdsasdasassdssdsd" + "somejunkasfdasfsafasafdsasdasassdssdsd" + "somejunkasfdasfsafasafdsasdasassdssdsd" + "somejunkasfdasfsafasafdsasdasassdssdsd"; + + uint32_t line_count = 0; + + MimeDecGetConfig()->decode_base64 = 1; + MimeDecGetConfig()->decode_quoted_printable = 1; + MimeDecGetConfig()->extract_urls = 1; + + /* Init parser */ + MimeDecParseState *state = MimeDecInitParser(&line_count, + TestDataChunkCallback); + + const char *str = "From: Sender1"; + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); + + str = "To: Recipient1"; + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); + + str = "Content-Type: text/plain"; + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); + + /* Contains 40 character filename */ + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)mimemsg, strlen(mimemsg), 1, state)); + + str = ""; + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); + + str = "A simple message line 1"; + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); + + /* Completed */ + FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseComplete(state)); + + MimeDecEntity *msg = state->msg; + FAIL_IF_NOT(msg); + + /* filename is not too long */ + FAIL_IF(msg->anomaly_flags & ANOM_LONG_FILENAME); + + MimeDecFreeEntity(msg); + + /* De Init parser */ + MimeDecDeInitParser(state); + + PASS; +} + #endif /* UNITTESTS */ void MimeDecRegisterTests(void) @@ -3048,5 +3214,7 @@ void MimeDecRegisterTests(void) UtRegisterTest("MimeIsExeURLTest01", MimeIsExeURLTest01); UtRegisterTest("MimeIsIpv4HostTest01", MimeIsIpv4HostTest01); UtRegisterTest("MimeIsIpv6HostTest01", MimeIsIpv6HostTest01); + UtRegisterTest("MimeDecParseLongFilename01", MimeDecParseLongFilename01); + UtRegisterTest("MimeDecParseLongFilename02", MimeDecParseLongFilename02); #endif /* UNITTESTS */ } diff --git a/src/util-decode-mime.h b/src/util-decode-mime.h index 1922264bee3..80bb96c2345 100644 --- a/src/util-decode-mime.h +++ b/src/util-decode-mime.h @@ -52,7 +52,7 @@ /* Anomaly Flags */ #define ANOM_INVALID_BASE64 1 /* invalid base64 chars */ -#define ANOM_INVALID_QP 2 /* invalid qouted-printable chars */ +#define ANOM_INVALID_QP 2 /* invalid quoted-printable chars */ #define ANOM_LONG_HEADER_NAME 4 /* header is abnormally long */ #define ANOM_LONG_HEADER_VALUE 8 /* header value is abnormally long * (includes multi-line) */ @@ -60,6 +60,7 @@ #define ANOM_LONG_ENC_LINE 32 /* Lines that exceed 76 octets */ #define ANOM_MALFORMED_MSG 64 /* Misc msg format errors found */ #define ANOM_LONG_BOUNDARY 128 /* Boundary too long */ +#define ANOM_LONG_FILENAME 256 /* filename truncated */ /* Publicly exposed size constants */ #define DATA_CHUNK_SIZE 3072 /* Should be divisible by 3 */ diff --git a/src/util-error.c b/src/util-error.c index 2b9d4014542..66f74c7de0f 100644 --- a/src/util-error.c +++ b/src/util-error.c @@ -45,6 +45,7 @@ const char * SCErrorToString(SCError err) CASE_CODE (SC_ERR_ACTION_ORDER); CASE_CODE (SC_ERR_PCRE_MATCH); CASE_CODE (SC_ERR_PCRE_GET_SUBSTRING); + CASE_CODE (SC_ERR_PCRE_COPY_SUBSTRING); CASE_CODE (SC_ERR_PCRE_COMPILE); CASE_CODE (SC_ERR_PCRE_STUDY); CASE_CODE (SC_ERR_PCRE_PARSE); diff --git a/src/util-error.h b/src/util-error.h index 0897b02651e..c8ef665e2a9 100644 --- a/src/util-error.h +++ b/src/util-error.h @@ -355,6 +355,7 @@ typedef enum { SC_ERR_DATASET, SC_WARN_ANOMALY_CONFIG, SC_WARN_ALERT_CONFIG, + SC_ERR_PCRE_COPY_SUBSTRING, SC_ERR_MAX } SCError; diff --git a/src/util-mpm.c b/src/util-mpm.c index f1b0b57329c..f7d0b1f46c4 100644 --- a/src/util-mpm.c +++ b/src/util-mpm.c @@ -45,6 +45,9 @@ #include "hs.h" #endif +MpmTableElmt mpm_table[MPM_TABLE_SIZE]; +int mpm_default_matcher; + /** * \brief Register a new Mpm Context. * diff --git a/src/util-mpm.h b/src/util-mpm.h index a47a9cf2e16..8f52fa68fcf 100644 --- a/src/util-mpm.h +++ b/src/util-mpm.h @@ -166,8 +166,8 @@ typedef struct MpmTableElmt_ { uint8_t flags; } MpmTableElmt; -MpmTableElmt mpm_table[MPM_TABLE_SIZE]; -int mpm_default_matcher; +extern MpmTableElmt mpm_table[MPM_TABLE_SIZE]; +extern int mpm_default_matcher; struct DetectEngineCtx_; diff --git a/src/util-privs.c b/src/util-privs.c index 5ce6843eb7d..bed5889b9ad 100644 --- a/src/util-privs.c +++ b/src/util-privs.c @@ -74,9 +74,10 @@ void SCDropMainThreadCaps(uint32_t userid, uint32_t groupid) CAP_NET_ADMIN, CAP_NET_RAW, CAP_SYS_NICE, -1); break; + case RUNMODE_NFLOG: case RUNMODE_NFQ: capng_updatev(CAPNG_ADD, CAPNG_EFFECTIVE|CAPNG_PERMITTED, - CAP_NET_ADMIN, /* needed for nfqueue inline mode */ + CAP_NET_ADMIN, /* needed for nflog and nfqueue inline mode */ CAP_SYS_NICE, -1); break; diff --git a/src/util-proto-name.c b/src/util-proto-name.c index 0b958884ca4..2392682d7b7 100644 --- a/src/util-proto-name.c +++ b/src/util-proto-name.c @@ -27,6 +27,9 @@ #include "suricata-common.h" #include "util-proto-name.h" +/** Lookup array to hold the information related to known protocol + * in /etc/protocols */ +char *known_proto[256]; static int init_once = 0; /** diff --git a/src/util-proto-name.h b/src/util-proto-name.h index e349a6db2eb..7cf69df928f 100644 --- a/src/util-proto-name.h +++ b/src/util-proto-name.h @@ -32,7 +32,7 @@ /** Lookup array to hold the information related to known protocol * in /etc/protocols */ -char *known_proto[256]; +extern char *known_proto[256]; uint8_t SCProtoNameValid(uint16_t); void SCProtoNameInit(void); diff --git a/src/util-spm.c b/src/util-spm.c index e6fcf3dda76..17bc1139e0e 100644 --- a/src/util-spm.c +++ b/src/util-spm.c @@ -59,6 +59,8 @@ #include "hs.h" #endif +SpmTableElmt spm_table[SPM_TABLE_SIZE]; + /** * \brief Returns the single pattern matcher algorithm to be used, based on the * spm-algo setting in yaml. diff --git a/src/util-spm.h b/src/util-spm.h index 2f3b60fe7ed..dbffb2bf304 100644 --- a/src/util-spm.h +++ b/src/util-spm.h @@ -71,7 +71,7 @@ typedef struct SpmTableElmt_ { const uint8_t *haystack, uint32_t haystack_len); } SpmTableElmt; -SpmTableElmt spm_table[SPM_TABLE_SIZE]; +extern SpmTableElmt spm_table[SPM_TABLE_SIZE]; void SpmTableSetup(void); diff --git a/suricata.yaml.in b/suricata.yaml.in index aacad7411d7..96b9b0b0d32 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -1338,6 +1338,10 @@ decoder: vxlan: enabled: true ports: $VXLAN_PORTS # syntax: '8472, 4789' + # ERSPAN Type I decode support + erspan: + typeI: + enabled: false ##