Give option for additional output JSON/Extended JSON

collector/dsc/Makefile.in

@@ -33,6 +33,7 @@ OBJS=\
 	client_ip_net_index.o \
 	server_ip_addr_index.o \
 	md_array_xml_printer.o \
+	md_array_json_printer.o \
 	ip_direction_index.o \
 	ip_proto_index.o \
 	ip_version_index.o \

collector/dsc/ParseConfig.cc

@@ -26,6 +26,7 @@ extern "C" int set_bpf_vlan_tag_byte_order(const char *);
 extern "C" int set_bpf_program(const char *);
 extern "C" int set_match_vlan(const char *);
 extern "C" int add_qname_filter(const char *name, const char *re);
+extern "C" int set_additional_output(const char *);
 
 extern "C" void ParseConfig(const char *);
 
@@ -44,6 +45,7 @@ Rule rHexPart;
 Rule rIPv6Address;
 Rule rIPAddress;
 Rule rHostOrNet;
+Rule rOutputs;
 
 Rule rInterface("Interface", 0);
 Rule rRunDir("RunDir", 0);
@@ -54,6 +56,7 @@ Rule rPacketFilterProg("PacketFilterProg", 0);
 Rule rDatasetOpt("DatasetOpt", 0);
 Rule rDataset("Dataset", 0);
 Rule rBVTBO("BVTBO", 0);
+Rule rAddOutput("AddOutput", 0);
 Rule rMatchVlan("MatchVlan", 0);
 Rule rQnameFilter("QnameFilter", 0);
 
@@ -152,6 +155,13 @@ interpret(const Pree &tree, int level)
 			return 0;
 		}
 	} else
+	if (tree.rid() == rAddOutput.id()) {
+		assert(tree.count() > 1);
+		if (set_additional_output(tree[1].image().c_str()) != 1) {
+			cerr << "interpret() failure in set_additional_output" << endl;
+			return 0;
+		}
+	} else
 	if (tree.rid() == rMatchVlan.id()) {
 		for(unsigned int i = 0; i<tree[1].count(); i++) {
 			if (set_match_vlan(tree[1][i].image().c_str()) != 1) {
@@ -206,7 +216,7 @@ ParseConfig(const char *fn)
 	rIPv6Address = rHexPart >> * ( ":" >> rIPv4Address );
 	rIPAddress = rIPv4Address | rIPv6Address;
 	rHostOrNet = string_r("host") | string_r("net");
-
+        rOutputs = string_r("json") | string_r("ext_json");
 
 	// rule/line level
 	rInterface = "interface" >>rBareToken >>";" ;
@@ -222,6 +232,7 @@ ParseConfig(const char *fn)
 		>>rBareToken
 		>>*rDatasetOpt >>";" ;
 	rBVTBO = "bpf_vlan_tag_byte_order" >>rHostOrNet >>";" ;
+        rAddOutput = "add_output" >>rOutputs >>";" ;
 	rMatchVlan = "match_vlan" >> +rDecimalNumber >>";" ;
 	rQnameFilter = "qname_filter" >>rBareToken >>rBareToken >>";" ;
 
@@ -235,6 +246,7 @@ ParseConfig(const char *fn)
 		rPacketFilterProg |
 		rDataset |
 		rBVTBO |
+        rAddOutput |
 		rMatchVlan |
 		rQnameFilter
 	) >> end_r;
@@ -252,6 +264,7 @@ ParseConfig(const char *fn)
 	rDecimalNumber.leaf(true);
 	rIPv4Address.leaf(true);
 	rHostOrNet.leaf(true);
+	rOutputs.leaf(true);
 
 	// commit points
         rInterface.committed(true);
@@ -261,6 +274,7 @@ ParseConfig(const char *fn)
         rPacketFilterProg.committed(true);
         rDataset.committed(true);
 	rBVTBO.committed(true);
+	rAddOutput.committed(true);
 	rMatchVlan.committed(true);
 
 	std::string config;

collector/dsc/config_hooks.c

@@ -15,6 +15,8 @@ void Ncap_init(const char *device, int promisc);
 #endif
 void Pcap_init(const char *device, int promisc);
 uint64_t minfree_bytes = 0;
+char output_json = 0;
+char output_ext_json = 0;
 
 int
 open_interface(const char *interface)
@@ -117,3 +119,33 @@ set_minfree_bytes(const char *s)
     minfree_bytes = strtoull(s, NULL, 10);
     return 1;
 }
+
+int check_unique_json_output(const char *output_format)
+{
+    if (output_json || output_ext_json) {
+        syslog(LOG_ERR, "add_output '%s' set but a json-like additional output already exists", output_format);
+        return 0;
+    }
+    return 1;
+}
+
+int
+set_additional_output(const char *output_format)
+{
+    syslog(LOG_INFO, "add_output %s", output_format);
+
+    if (0 == strcmp(output_format, "json")) {
+        if (!check_unique_json_output(output_format))
+            return 0;
+        output_json = 1;
+        return 1;
+    }
+    if (0 == strcmp(output_format, "ext_json")) {
+        if (!check_unique_json_output(output_format))
+            return 0;
+        output_ext_json = 1;
+        return 1;
+    }
+    syslog(LOG_ERR, "unknown add_output '%s'", output_format);
+    return 0;
+}

collector/dsc/daemon.c

@@ -52,7 +52,12 @@ extern void country_indexer_init(void);
 #endif
 extern void ParseConfig(const char *);
 extern uint64_t minfree_bytes;
+extern char output_json;
+extern char output_ext_json;
 extern int n_pcap_offline;
+extern md_array_printer xml_printer;
+extern md_array_printer json_printer;
+extern md_array_printer ext_json_printer;
 
 void
 daemonize(void)
@@ -134,21 +139,21 @@ usage(void)
 }
 
 static int
-dump_reports(void)
+dump_report(char *extension, char *start_file, char *end_file, md_array_printer * printer)
 {
     int fd;
     FILE *fp;
     char fname[128];
     char tname[128];
 
     if (disk_is_full()) {
-	syslog(LOG_NOTICE, "%s", "Not enough free disk space to write XML files");
+	syslog(LOG_NOTICE, "Not enough free disk space to write '%s' report files", extension);
 	return 1;
     }
 #if HAVE_LIBNCAP
-    snprintf(fname, 128, "%d.dscdata.xml", Ncap_finish_time());
+    snprintf(fname, 128, "%d.dscdata.%s", Ncap_finish_time(), extension);
 #else
-    snprintf(fname, 128, "%d.dscdata.xml", Pcap_finish_time());
+    snprintf(fname, 128, "%d.dscdata.%s", Pcap_finish_time(), extension);
 #endif
     snprintf(tname, 128, "%s.XXXXXXXXX", fname);
     fd = mkstemp(tname);
@@ -164,11 +169,14 @@ dump_reports(void)
     }
     if (debug_flag)
 	fprintf(stderr, "writing to %s\n", tname);
-    fprintf(fp, "<dscdata>\n");
+
+    fprintf(fp, start_file);
+
     /* amalloc_report(); */
-    pcap_report(fp);
-    dns_message_report(fp);
-    fprintf(fp, "</dscdata>\n");
+    pcap_report(fp, printer);
+    dns_message_report(fp, printer);
+
+    fprintf(fp, end_file);
 
     /*
      * XXX need chmod because files are written as root, but may be processed
@@ -178,10 +186,24 @@ dump_reports(void)
     fclose(fp);
     if (debug_flag)
 	fprintf(stderr, "renaming to %s\n", fname);
+
     rename(tname, fname);
     return 0;
 }
 
+static int
+dump_reports(void)
+{
+    int err = dump_report("xml", "<dscdata>\n", "</dscdata>\n", &xml_printer);
+
+    if (output_json)
+	err = err & dump_report("json", "{\n", "\n}\n", &json_printer);
+    else if (output_ext_json)
+	err = err & dump_report("json", "{\n", "\n}\n", &ext_json_printer);
+
+    return err;
+}
+
 int
 main(int argc, char *argv[])
 {

collector/dsc/dns_message.c

@@ -18,7 +18,6 @@
 
 #include "xmalloc.h"
 #include "dns_message.h"
-#include "md_array.h"
 #include "null_index.h"
 #include "qtype_index.h"
 #include "qclass_index.h"
@@ -53,7 +52,6 @@
 
 #include "syslog_debug.h"
 
-extern md_array_printer xml_printer;
 extern int debug_flag;
 static md_array_list *Arrays = NULL;
 static filter_list *DNSFilters = NULL;
@@ -348,11 +346,12 @@ dns_message_add_array(const char *name, const char *fn, const char *fi,
 }
 
 void
-dns_message_report(FILE * fp)
+dns_message_report(FILE *fp, md_array_printer *printer)
 {
     md_array_list *a;
-    for (a = Arrays; a; a = a->next)
-	md_array_print(a->theArray, &xml_printer, fp);
+    for (a = Arrays; a; a = a->next) {
+        md_array_print(a->theArray, printer, fp);
+    }
 }
 
 void

collector/dsc/dns_message.h

@@ -7,6 +7,7 @@
 
 #include "inX_addr.h"
 #include "dataset_opt.h"
+#include "md_array.h"
 
 #define MAX_QNAME_SZ 512
 
@@ -49,11 +50,10 @@ struct _dns_message
     /* ... */
 };
 
-void dns_message_report(FILE *);
-int dns_message_add_array(const char *, const char *, const char *, const char *, const char *, const char *,
-    dataset_opt);
-const char *dns_message_QnameToNld(const char *, int);
-const char *dns_message_tld(dns_message * m);
+void dns_message_report(FILE *, md_array_printer *);
+int dns_message_add_array(const char *, const char *,const char *,const char *,const char *,const char *, dataset_opt);
+const char * dns_message_QnameToNld(const char *, int);
+const char * dns_message_tld(dns_message * m);
 void dns_message_init(void);
 void dns_message_clear_arrays(void);
 void dns_message_handle(dns_message *);

collector/dsc/dsc.conf.sample

@@ -22,6 +22,16 @@ run_dir "/usr/local/dsc/run/NODENAME";
 #
 minfree_bytes 5000000;
 
+
+# add_output
+#       Set an additional output format. Files with this format 
+#       will be generated along with the XML files.
+#       Currently, the available additional outputs are json and
+#       ext_json. At most one add_output can be set.
+#
+# add_output json;
+
+
 # pid_file
 #
 #	filename where DSC should store its process-id

collector/dsc/md_array.c

@@ -7,7 +7,6 @@
 #include <syslog.h>
 
 #include "xmalloc.h"
-#include "dataset_opt.h"
 #include "md_array.h"
 #include "dns_message.h"
 #include "pcap.h"
@@ -214,7 +213,7 @@ md_array_print(md_array * a, md_array_printer * pr, FILE * fp)
 	nvals = a->d2.alloc_sz;
 	sortme = xcalloc(nvals, sizeof(*sortme));
 	if (NULL == sortme) {
-	    syslog(LOG_CRIT, "%s", "Cant output XML file chunk due to malloc failure!");
+	    syslog(LOG_CRIT, "%s", "Cant output file chunk due to malloc failure!");
 	    continue;		/* OUCH! */
 	}
 	while ((i2 = a->d2.indexer->iter_fn(&label2)) > -1) {

collector/dsc/md_array.h

@@ -1,4 +1,6 @@
-
+#ifndef MD_ARRAY_H
+#define MD_ARRAY_H
+#include "dataset_opt.h"
 
 typedef struct _md_array md_array;
 typedef struct _md_array_printer md_array_printer;
@@ -80,3 +82,5 @@ md_array *md_array_create(const char *name, filter_list *, const char *, indexer
 int md_array_print(md_array * a, md_array_printer * pr, FILE * fp);
 filter_list **md_array_filter_list_append(filter_list ** fl, FLTR * f);
 FLTR *md_array_create_filter(const char *name, filter_func *, const void *context);
+
+#endif