From 7e7b248f6520c14361984aef7a78203b06d007eb Mon Sep 17 00:00:00 2001 From: fernandomariano Date: Wed, 17 Apr 2019 13:35:51 -0600 Subject: [PATCH 1/5] changing default elasticsearch version to 5 in order to work smoothly --- nxapi/nxapi.json | 2 +- nxapi/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nxapi/nxapi.json b/nxapi/nxapi.json index 83a246c6..1b8b7e71 100644 --- a/nxapi/nxapi.json +++ b/nxapi/nxapi.json @@ -8,7 +8,7 @@ "doctype" : "events", "default_ttl" : "7200", "max_size" : "1000", - "version" : "2" + "version" : "5" }, "syslogd": { "host" : "0.0.0.0", diff --git a/nxapi/requirements.txt b/nxapi/requirements.txt index 9ab72d3b..9619d49c 100755 --- a/nxapi/requirements.txt +++ b/nxapi/requirements.txt @@ -1,2 +1,2 @@ -elasticsearch +elasticsearch==5.5.3 GeoIP From f2a629240add1d5c23272194faf105c0e7db797f Mon Sep 17 00:00:00 2001 From: fernandomariano Date: Wed, 17 Apr 2019 13:37:16 -0600 Subject: [PATCH 2/5] added countries for stats reporting and generating whitelist process --- nxapi/nxapi/nxparse.py | 6 ++++-- nxapi/nxapi/nxtransform.py | 7 +++++-- nxapi/nxtool.py | 7 +++++++ 3 files changed, 16 insertions(+), 4 deletions(-) mode change 100644 => 100755 nxapi/nxapi/nxtransform.py diff --git a/nxapi/nxapi/nxparse.py b/nxapi/nxapi/nxparse.py index e390288c..4ace6814 100755 --- a/nxapi/nxapi/nxparse.py +++ b/nxapi/nxapi/nxparse.py @@ -433,7 +433,8 @@ def set_mappings(self): "zone" : {"type": "keyword"}, "server" : {"type": "keyword"}, "whitelisted" : {"type" : "keyword"}, - "ip" : {"type" : "keyword"} + "ip" : {"type" : "keyword"}, + "country" : {"type" : "keyword"} } } }) @@ -470,7 +471,8 @@ def set_mappings(self): "server" : {"type": "string", "index":"not_analyzed"}, "whitelisted" : {"type" : "string", "index":"not_analyzed"}, "content" : {"type" : "string", "index":"not_analyzed"}, - "ip" : { "type" : "string", "index":"not_analyzed"} + "ip" : { "type" : "string", "index":"not_analyzed"}, + "country" : { "type" : "string", "index":"not_analyzed"} } } }) diff --git a/nxapi/nxapi/nxtransform.py b/nxapi/nxapi/nxtransform.py old mode 100644 new mode 100755 index d34603d8..930b7007 --- a/nxapi/nxapi/nxtransform.py +++ b/nxapi/nxapi/nxtransform.py @@ -252,7 +252,7 @@ def fancy_display(self, full_wl, scores, template=None): output.append("#Rule ({0}) {1}\n".format(rid, self.core_msg.get(rid, 'Unknown ..'))) if self.cfg["output"]["verbosity"] >= 4: output.append("#total hits {0}\n".format(full_wl['total_hits'])) - for x in ["content", "peers", "uri", "var_name"]: + for x in ["content", "peers", "country", "uri", "var_name"]: if x not in full_wl.keys(): continue for y in full_wl[x]: @@ -721,12 +721,15 @@ def gen_wl(self, tpl, rule={}): if res['hits']['total'] > 0: clist = [] peers = [] + country = [] uri = [] var_name = [] for x in res['hits']['hits']: if len(x.get("_source").get("ip", "")) > 0 and x.get("_source").get("ip", "") not in peers: peers.append(x["_source"]["ip"]) + if len(x.get("_source").get("country", "")) > 0 and x.get("_source").get("country", "") not in country: + country.append(x["_source"]["country"]) if len(x.get("_source").get("uri", "")) > 0 and x.get("_source").get("uri", "") not in uri: uri.append(x["_source"]["uri"]) if len(x.get("_source").get("var_name", "")) > 0 and x.get("_source").get("var_name", "") not in var_name: @@ -735,7 +738,7 @@ def gen_wl(self, tpl, rule={}): clist.append(x["_source"]["content"]) if len(clist) >= 5: break - retlist.append({'rule' : rule, 'content' : clist[:5], 'total_hits' : res['hits']['total'], 'peers' : peers[:5], 'uri' : uri[:5], + retlist.append({'rule' : rule, 'content' : clist[:5], 'total_hits' : res['hits']['total'], 'peers' : peers[:5], 'country' : country[:5], 'uri' : uri[:5], 'var_name' : var_name[:5]}) return retlist return [] diff --git a/nxapi/nxtool.py b/nxapi/nxtool.py index 01a75454..06b17c34 100755 --- a/nxapi/nxtool.py +++ b/nxapi/nxtool.py @@ -311,6 +311,13 @@ def get_filter(arg_filter): logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3])) except: logging.warning("--malformed--") + logging.info(translate.red.format("# Top Country(ies) :")) + for e in translate.fetch_top(cfg.cfg["global_filters"], "country", limit=10): + try: + list_e = e.split() + logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3])) + except: + logging.warning("--malformed--") sys.exit(0) From 2dd81f8ab94d43ee2ba03effda2181e4833a7b81 Mon Sep 17 00:00:00 2001 From: fernandomariano Date: Wed, 17 Apr 2019 14:14:10 -0600 Subject: [PATCH 3/5] fixing fielddata id on elasticsearch --- nxapi/nxapi/nxparse.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nxapi/nxapi/nxparse.py b/nxapi/nxapi/nxparse.py index 4ace6814..bbb6e333 100755 --- a/nxapi/nxapi/nxparse.py +++ b/nxapi/nxapi/nxparse.py @@ -428,7 +428,9 @@ def set_mappings(self): # That is why time based indexes are recommended over this sort of thing and why # _ttl was deprecated in the first place) #"_ttl" : { "enabled" : "true", "default" : "4d" }, - "properties" : { "var_name" : {"type": "keyword"}, + "properties" : { + "id" : {"type": "keyword"}, + "var_name" : {"type": "keyword"}, "uri" : {"type": "keyword"}, "zone" : {"type": "keyword"}, "server" : {"type": "keyword"}, @@ -465,7 +467,9 @@ def set_mappings(self): body={ "events" : { "_ttl" : { "enabled" : "true", "default" : "4d" }, - "properties" : { "var_name" : {"type": "string", "index":"not_analyzed"}, + "properties" : { + "id" : {"type": "string", "index":"not_analyzed"}, + "var_name" : {"type": "string", "index":"not_analyzed"}, "uri" : {"type": "string", "index":"not_analyzed"}, "zone" : {"type": "string", "index":"not_analyzed"}, "server" : {"type": "string", "index":"not_analyzed"}, From ad414111d7a464fe9b2a282ba3e17b7b7bb5ff97 Mon Sep 17 00:00:00 2001 From: fernandomariano Date: Wed, 17 Apr 2019 14:22:45 -0600 Subject: [PATCH 4/5] changing logging to the previous behavior (print command) - no formatting output --- nxapi/nxtool.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nxapi/nxtool.py b/nxapi/nxtool.py index 06b17c34..cf5f203b 100755 --- a/nxapi/nxtool.py +++ b/nxapi/nxtool.py @@ -27,8 +27,8 @@ # Initialize logging logging.basicConfig(stream=sys.stdout, level=logging.INFO, - format='%(asctime)s - %(levelname)s: %(message)s (%(name)s)', - datefmt='%c') + format=None, + datefmt=None) def open_fifo(fifo): try: @@ -158,7 +158,7 @@ def get_filter(arg_filter): use_ssl = bool(cfg.cfg["elastic"]["use_ssl"]) except KeyError: use_ssl = False - + es = elasticsearch.Elasticsearch(cfg.cfg["elastic"]["host"], use_ssl=use_ssl) # Get ES version from the client and avail it at cfg es_version = es.info()['version'].get('number', None) @@ -182,7 +182,7 @@ def get_filter(arg_filter): results = translate.full_auto() if results: for result in results: - logging.debug("{0}".format(result)) + logging.info("{0}".format(result)) else: logging.critical("No hits for this filter.") sys.exit(1) From d6eb85e834106575cb078619ef34c393987b352e Mon Sep 17 00:00:00 2001 From: fernandomariano Date: Wed, 17 Apr 2019 14:25:26 -0600 Subject: [PATCH 5/5] added sample nginx log file --- nxapi/sample/nginx.log | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 nxapi/sample/nginx.log diff --git a/nxapi/sample/nginx.log b/nxapi/sample/nginx.log new file mode 100644 index 00000000..83acd7b0 --- /dev/null +++ b/nxapi/sample/nginx.log @@ -0,0 +1,4 @@ +2019/04/17 11:37:19 [error] 11495#11495: *323360 NAXSI_EXLOG: ip=172.18.13.136&server=myexample.org&uri=%2F&id=1302&zone=ARGS&var_name=q&content=%3C%3E, client: 172.18.13.136, server: myexample.org, request: "GET /?q=%3C%3E HTTP/2.0", host: "myexample.org" +2019/04/17 11:37:19 [error] 11495#11495: *323360 NAXSI_FMT: ip=172.18.13.136&server=myexample.org&uri=/&learning=0&vers=0.56&total_processed=5&total_blocked=1&block=1&cscore0=$XSS&score0=8&zone0=ARGS&id0=1302&var_name0=q, client: 172.18.13.136, server: myexample.org, request: "GET /?q=%3C%3E HTTP/2.0", host: "myexample.org" +2019/04/17 11:37:46 [error] 11495#11495: *323360 NAXSI_EXLOG: ip=216.208.239.171&server=myexample.org&uri=%2F&id=1000&zone=ARGS&var_name=q&content=%22update%20table%20%28%29%22, client: 216.208.239.171, server: myexample.org, request: "GET /?q=%22update%20table%20()%22 HTTP/2.0", host: "myexample.org" +2019/04/17 11:37:46 [error] 11495#11495: *323360 NAXSI_FMT: ip=216.208.239.171&server=myexample.org&uri=/&learning=0&vers=0.56&total_processed=7&total_blocked=2&block=1&cscore0=$SQL&score0=8&zone0=ARGS&id0=1000&var_name0=q, client: 216.208.239.171, server: myexample.org, request: "GET /?q=%22update%20table%20()%22 HTTP/2.0", host: "myexample.org"