Merge remote-tracking branch 'origin/topic/timw/150-to-json'

* origin/topic/timw/150-to-json:
  Update submodules for JSON work
  Update unit tests for JSON logger to match new output
  Modify JSON log writer to use the external JSON library
  Update unit test output to match json.zeek being deprecated and slight format changes to JSON output
  Add proper JSON serialization via C++, deprecate json.zeek
  Add new method for escaping UTF8 strings for JSON output
  Move do_sub method from zeek.bif to StringVal class method
  Move record_fields method from zeek.bif to Val class method
  Add ToStdString method for StringVal

CHANGES

@@ -1,4 +1,11 @@
 
+2.6-586 | 2019-07-11 11:15:40 -0700
+
+  * Convert all JSON output to use an external library for better consistency (Tim Wojtulewicz, Corelight)
+
+    See NEWS for more details; this makes to_json a bif and causes slight changes in its
+    output, as well as the output of the JSON logger.
+
 2.6-576 | 2019-07-10 18:38:54 -0700
 
   * Remove unused option: chunked_io_buffer_soft_cap (Jon Siwek, Corelight)

NEWS

@@ -347,6 +347,16 @@ Changed Functionality
   of each other on separate cluster nodes to all be logged rather
   than suppressed and de-duplicated into a single notice.
 
+
+- to_json is now a bif, no longer a script. Loading base/utils/json.zeek is no
+  longer necessary and has been deprecated. to_json should yield much better, always
+  valid json. There are some small differences in output; unnecessary spaces are removed
+  and port values are rendered differently, now including the port and the protocol.
+
+- The output of the JSON logger now uses an external library to generate json. There
+  are small changes to the output; most visibly double numbers are now rounded slightly
+  differently. The way in which port values are rendered does _not_ change for JSON logs.
+
 Removed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-2.6-576
+2.6-586

scripts/base/frameworks/openflow/plugins/ryu.zeek

@@ -3,7 +3,6 @@
 @load base/frameworks/openflow
 @load base/utils/active-http
 @load base/utils/exec
-@load base/utils/json
 
 module OpenFlow;
 

scripts/base/utils/json.zeek

@@ -1,109 +1,2 @@
-##! Functions to assist with generating JSON data from Zeek data scructures.
-# We might want to implement this in core somtime, this looks... hacky at best.
-
-@load base/utils/strings
-
-## A function to convert arbitrary Zeek data into a JSON string.
-##
-## v: The value to convert to JSON.  Typically a record.
-##
-## only_loggable: If the v value is a record this will only cause
-##                fields with the &log attribute to be included in the JSON.
-##
-## returns: a JSON formatted string.
-function to_json(v: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/): string
-	{
-	local tn = type_name(v);
-	switch ( tn )
-		{
-		case "type":
-		return "";
-
-		case "string":
-		return cat("\"", gsub(gsub(clean(v), /\\/, "\\\\"), /\"/, "\\\""), "\"");
-
-		case "port":
-		return cat(port_to_count(to_port(cat(v))));
-
-		case "enum":
-		fallthrough;
-		case "interval":
-		fallthrough;
-		case "addr":
-		fallthrough;
-		case "subnet":
-		return cat("\"", v, "\"");
-
-		case "int":
-		fallthrough;
-		case "count":
-		fallthrough;
-		case "time":
-		return cat(v);
-
-		case "double":
-		return fmt("%.16g", v);
-
-		case "bool":
-		local bval: bool = v;
-		return bval ? "true" : "false";
-
-		default:
-		break;
-		}
-
-	if ( /^record/ in tn )
-		{
-		local rec_parts: string_vec = vector();
-
-		local ft = record_fields(v);
-		for ( field, field_desc in ft )
-			{
-			# replace the escape pattern in the field.
-			if( field_escape_pattern in field )
-				field = cat(sub(field, field_escape_pattern, ""));
-			if ( field_desc?$value && (!only_loggable || field_desc$log) )
-				{
-				local onepart = cat("\"", field, "\": ", to_json(field_desc$value, only_loggable));
-				rec_parts += onepart;
-				}
-			}
-			return cat("{", join_string_vec(rec_parts, ", "), "}");
-		}
-
-	# None of the following are supported.
-	else if ( /^set/ in tn )
-		{
-		local set_parts: string_vec = vector();
-		local sa: set[bool] = v;
-		for ( sv in sa )
-			{
-			set_parts += to_json(sv, only_loggable);
-			}
-		return cat("[", join_string_vec(set_parts, ", "), "]");
-		}
-	else if ( /^table/ in tn )
-		{
-		local tab_parts: vector of string = vector();
-		local ta: table[bool] of any = v;
-		for ( ti, tv in ta )
-			{
-			local ts = to_json(ti);
-			local if_quotes = (ts[0] == "\"") ? "" : "\"";
-			tab_parts += cat(if_quotes, ts, if_quotes, ": ", to_json(tv, only_loggable));
-			}
-		return cat("{", join_string_vec(tab_parts, ", "), "}");
-		}
-	else if ( /^vector/ in tn )
-		{
-		local vec_parts: string_vec = vector();
-		local va: vector of any = v;
-		for ( vi in va )
-			{
-			vec_parts += to_json(va[vi], only_loggable);
-			}
-		return cat("[", join_string_vec(vec_parts, ", "), "]");
-		}
-
-	return "\"\"";
-	}
+## This file is deprecated in favor of to_json in zeek.bif
+@deprecated="Remove in 3.1.  to_json is now always available as a built-in function."