Merge remote-tracking branch 'origin/topic/robin/gh-623-sampling'

- Merge adjustments:
  - Minor whitespace/style tweaks
  - Fixed portability of the btest due to differences in `uniq -c`
    output format

* origin/topic/robin/gh-623-sampling:
  Extend weird sampling with option to track selected weirds globally.

CHANGES

@@ -1,4 +1,11 @@
 
+3.3.0-dev.252 | 2020-09-08 17:04:19 -0700
+
+  * GH-623: Extend weird sampling with option to track selected weirds globally. (Robin Sommer, Corelight)
+
+    The new set "sampling_global_list" lists weirds to rate-limit
+    globally instead of per connection/flow.
+
 3.3.0-dev.249 | 2020-09-04 18:30:19 -0700
 
   * Exclude installing "zeek -> ." include dir symlink (Jon Siwek, Corelight)

VERSION

@@ -1 +1 @@
-3.3.0-dev.249
+3.3.0-dev.252

scripts/base/init-bare.zeek

@@ -5263,6 +5263,9 @@ export {
 	## Prevents rate-limiting sampling of any weirds named in the table.
 	option sampling_whitelist: set[string] = {};
 
+	## Rate-limits weird names in the table globally instead of per connection/flow.
+	option sampling_global_list: set[string] = {};
+
 	## How many weirds of a given type to tolerate before sampling begins.
 	## I.e. this many consecutive weirds of a given type will be allowed to
 	## raise events for script-layer handling before being rate-limited.

src/Reporter.cc

@@ -72,20 +72,27 @@ void Reporter::InitOptions()
 	weird_sampling_rate = id::find_val("Weird::sampling_rate")->AsCount();
 	weird_sampling_threshold = id::find_val("Weird::sampling_threshold")->AsCount();
 	weird_sampling_duration = id::find_val("Weird::sampling_duration")->AsInterval();
-	auto wl_val = id::find_val("Weird::sampling_whitelist")->AsTableVal();
-	auto wl_table = wl_val->AsTable();
 
-	detail::HashKey* k;
-	IterCookie* c = wl_table->InitForIteration();
-	TableEntryVal* v;
-
-	while ( (v = wl_table->NextEntry(k, c)) )
+	auto init_weird_set = [](WeirdSet* set, const char* name)
 		{
-		auto index = wl_val->RecreateIndex(*k);
-		std::string key = index->Idx(0)->AsString()->CheckString();
-		weird_sampling_whitelist.emplace(move(key));
-		delete k;
-		}
+		auto wl_val = id::find_val(name)->AsTableVal();
+		auto wl_table = wl_val->AsTable();
+
+		detail::HashKey* k;
+		IterCookie* c = wl_table->InitForIteration();
+		TableEntryVal* v;
+
+		while ( (v = wl_table->NextEntry(k, c)) )
+			{
+			auto index = wl_val->RecreateIndex(*k);
+			std::string key = index->Idx(0)->AsString()->CheckString();
+			set->emplace(move(key));
+			delete k;
+			}
+		};
+
+	init_weird_set(&weird_sampling_whitelist, "Weird::sampling_whitelist");
+	init_weird_set(&weird_sampling_global_list, "Weird::sampling_global_list");
 	}
 
 void Reporter::Info(const char* fmt, ...)
@@ -307,6 +314,18 @@ void Reporter::ResetExpiredConnWeird(const ConnTuple& id)
 	expired_conn_weird_state.erase(id);
 	}
 
+Reporter::PermitWeird Reporter::CheckGlobalWeirdLists(const char* name)
+	{
+	if ( WeirdOnSamplingWhiteList(name) )
+		return PermitWeird::Allow;
+
+	if ( WeirdOnGlobalList(name) )
+		// We track weirds on the global list through the "net_weird" table.
+		return PermitNetWeird(name) ? PermitWeird::Allow : PermitWeird::Deny;
+
+	return PermitWeird::Unknown;
+	}
+
 bool Reporter::PermitNetWeird(const char* name)
 	{
 	auto& count = net_weird_state[name];
@@ -395,12 +414,16 @@ void Reporter::Weird(file_analysis::File* f, const char* name, const char* addl)
 	{
 	UpdateWeirdStats(name);
 
-	if ( ! WeirdOnSamplingWhiteList(name) )
-		{
+	switch ( CheckGlobalWeirdLists(name) ) {
+	case PermitWeird::Allow:
+		break;
+	case PermitWeird::Deny:
+		return;
+	case PermitWeird::Unknown:
 		if ( ! f->PermitWeird(name, weird_sampling_threshold,
 		                      weird_sampling_rate, weird_sampling_duration) )
 			return;
-		}
+	}
 
 	WeirdHelper(file_weird, {f->ToVal()->Ref(), new StringVal(addl)},
 	            "%s", name);
@@ -410,12 +433,16 @@ void Reporter::Weird(Connection* conn, const char* name, const char* addl)
 	{
 	UpdateWeirdStats(name);
 
-	if ( ! WeirdOnSamplingWhiteList(name) )
-		{
+	switch ( CheckGlobalWeirdLists(name) ) {
+	case PermitWeird::Allow:
+		break;
+	case PermitWeird::Deny:
+		return;
+	case PermitWeird::Unknown:
 		if ( ! conn->PermitWeird(name, weird_sampling_threshold,
 		                         weird_sampling_rate, weird_sampling_duration) )
 			return;
-		}
+	}
 
 	WeirdHelper(conn_weird, {conn->ConnVal()->Ref(), new StringVal(addl)},
 	            "%s", name);
@@ -426,11 +453,15 @@ void Reporter::Weird(RecordValPtr conn_id, StringValPtr uid,
 	{
 	UpdateWeirdStats(name);
 
-	if ( ! WeirdOnSamplingWhiteList(name) )
-		{
+	switch ( CheckGlobalWeirdLists(name) ) {
+	case PermitWeird::Allow:
+		break;
+	case PermitWeird::Deny:
+		return;
+	case PermitWeird::Unknown:
 		if ( ! PermitExpiredConnWeird(name, *conn_id) )
 			return;
-		}
+	}
 
 	WeirdHelper(expired_conn_weird,
 	            {conn_id.release(), uid.release(), new StringVal(addl)},
@@ -441,11 +472,15 @@ void Reporter::Weird(const IPAddr& orig, const IPAddr& resp, const char* name, c
 	{
 	UpdateWeirdStats(name);
 
-	if ( ! WeirdOnSamplingWhiteList(name) )
-		{
+	switch ( CheckGlobalWeirdLists(name) ) {
+	case PermitWeird::Allow:
+		break;
+	case PermitWeird::Deny:
+		return;
+	case PermitWeird::Unknown:
 		if ( ! PermitFlowWeird(name, orig, resp) )
 			 return;
-		}
+	}
 
 	WeirdHelper(flow_weird,
 	            {new AddrVal(orig), new AddrVal(resp), new StringVal(addl)},

src/Reporter.h

@@ -190,6 +190,24 @@ class Reporter {
 		this->weird_sampling_whitelist = weird_sampling_whitelist;
 		}
 
+	/**
+	 * Gets the weird sampling global list.
+	 */
+	WeirdSet GetWeirdSamplingGlobalList() const
+		{
+		return weird_sampling_global_list;
+		}
+
+	/**
+	 * Sets the weird sampling global list.
+	 *
+	 * @param weird_sampling_global list New weird sampling global list.
+	 */
+	void SetWeirdSamplingGlobalList(const WeirdSet& weird_sampling_global_list)
+		{
+		this->weird_sampling_global_list = weird_sampling_global_list;
+		}
+
 	/**
 	 * Gets the current weird sampling threshold.
 	 *
@@ -269,10 +287,15 @@ class Reporter {
 	void UpdateWeirdStats(const char* name);
 	inline bool WeirdOnSamplingWhiteList(const char* name)
 		{ return weird_sampling_whitelist.find(name) != weird_sampling_whitelist.end(); }
+	inline bool WeirdOnGlobalList(const char* name)
+		{ return weird_sampling_global_list.find(name) != weird_sampling_global_list.end(); }
 	bool PermitNetWeird(const char* name);
 	bool PermitFlowWeird(const char* name, const IPAddr& o, const IPAddr& r);
 	bool PermitExpiredConnWeird(const char* name, const RecordVal& conn_id);
 
+	enum class PermitWeird { Allow, Deny, Unknown };
+	PermitWeird CheckGlobalWeirdLists(const char* name);
+
 	bool EmitToStderr(bool flag)
 		{ return flag || ! after_zeek_init; }
 
@@ -294,6 +317,7 @@ class Reporter {
 	WeirdConnTupleMap expired_conn_weird_state;
 
 	WeirdSet weird_sampling_whitelist;
+	WeirdSet weird_sampling_global_list;
 	uint64_t weird_sampling_threshold;
 	uint64_t weird_sampling_rate;
 	double weird_sampling_duration;

src/reporter.bif

@@ -178,7 +178,7 @@ function Reporter::get_weird_sampling_whitelist%(%): string_set
 ##
 ## Returns: Always true.
 function Reporter::set_weird_sampling_whitelist%(weird_sampling_whitelist: string_set%) : bool
-%{
+	%{
 	auto wl_val = weird_sampling_whitelist->AsTableVal();
 	auto wl_table = wl_val->AsTable();
 	std::unordered_set<std::string> whitelist_set;
@@ -196,7 +196,47 @@ function Reporter::set_weird_sampling_whitelist%(weird_sampling_whitelist: strin
 		}
 	reporter->SetWeirdSamplingWhitelist(whitelist_set);
 	return zeek::val_mgr->True();
-%}
+	%}
+
+## Gets the weird sampling global list
+##
+## Returns: Current weird sampling global list
+function Reporter::get_weird_sampling_global_list%(%): string_set
+	%{
+	auto set = zeek::make_intrusive<zeek::TableVal>(zeek::id::string_set);
+	for ( auto el : reporter->GetWeirdSamplingGlobalList() )
+		{
+		auto idx = zeek::make_intrusive<zeek::StringVal>(el);
+		set->Assign(std::move(idx), nullptr);
+		}
+	return set;
+	%}
+
+## Sets the weird sampling global list
+##
+## global_list: New weird sampling rate.
+##
+## Returns: Always true.
+function Reporter::set_weird_sampling_global_list%(weird_sampling_global_list: string_set%) : bool
+	%{
+	auto wl_val = weird_sampling_global_list->AsTableVal();
+	auto wl_table = wl_val->AsTable();
+	std::unordered_set<std::string> global_list_set;
+
+	zeek::detail::HashKey* k;
+	IterCookie* c = wl_table->InitForIteration();
+	TableEntryVal* v;
+
+	while ( (v = wl_table->NextEntry(k, c)) )
+		{
+		auto index = wl_val->RecreateIndex(*k);
+		string key = index->Idx(0)->AsString()->CheckString();
+		global_list_set.emplace(move(key));
+		delete k;
+		}
+	reporter->SetWeirdSamplingGlobalList(global_list_set);
+	return zeek::val_mgr->True();
+	%}
 
 ## Gets the current weird sampling threshold
 ##

testing/btest/Baseline/core.reporter-weird-sampling-global/output

@@ -0,0 +1,6 @@
+10 conn_weird, global_listed_conn_weird
+20 conn_weird, my_conn_weird
+10 flow_weird, global_listed_flow_weird
+20 flow_weird, my_flow_weird
+10 net_weird, global_listed_net_weird
+10 net_weird, my_net_weird