Skip to content
Permalink
Browse files

Fixed issues to setup nxapi on ES5 and added country location on stat…

…s and generated whitelists (#467)

* changing default elasticsearch version to 5 in order to work smoothly

* added countries for stats reporting and generating whitelist process

* fixing fielddata id on elasticsearch

* changing logging to the previous behavior (print command) - no formatting output

* added sample nginx log file
  • Loading branch information...
fernandomariano authored and he2ss committed May 7, 2019
1 parent 450313b commit d3cbf8a5b055c3d6ae17be37797ecb82d0a4956c
Showing with 32 additions and 12 deletions.
  1. +1 −1 nxapi/nxapi.json
  2. +10 −4 nxapi/nxapi/nxparse.py
  3. +5 −2 nxapi/nxapi/nxtransform.py
  4. +11 −4 nxapi/nxtool.py
  5. +1 −1 nxapi/requirements.txt
  6. +4 −0 nxapi/sample/nginx.log
@@ -8,7 +8,7 @@
"doctype" : "events",
"default_ttl" : "7200",
"max_size" : "1000",
"version" : "2"
"version" : "5"
},
"syslogd": {
"host" : "0.0.0.0",
@@ -428,12 +428,15 @@ def set_mappings(self):
# That is why time based indexes are recommended over this sort of thing and why
# _ttl was deprecated in the first place)
#"_ttl" : { "enabled" : "true", "default" : "4d" },
"properties" : { "var_name" : {"type": "keyword"},
"properties" : {
"id" : {"type": "keyword"},
"var_name" : {"type": "keyword"},
"uri" : {"type": "keyword"},
"zone" : {"type": "keyword"},
"server" : {"type": "keyword"},
"whitelisted" : {"type" : "keyword"},
"ip" : {"type" : "keyword"}
"ip" : {"type" : "keyword"},
"country" : {"type" : "keyword"}
}
}
})
@@ -464,13 +467,16 @@ def set_mappings(self):
body={
"events" : {
"_ttl" : { "enabled" : "true", "default" : "4d" },
"properties" : { "var_name" : {"type": "string", "index":"not_analyzed"},
"properties" : {
"id" : {"type": "string", "index":"not_analyzed"},
"var_name" : {"type": "string", "index":"not_analyzed"},
"uri" : {"type": "string", "index":"not_analyzed"},
"zone" : {"type": "string", "index":"not_analyzed"},
"server" : {"type": "string", "index":"not_analyzed"},
"whitelisted" : {"type" : "string", "index":"not_analyzed"},
"content" : {"type" : "string", "index":"not_analyzed"},
"ip" : { "type" : "string", "index":"not_analyzed"}
"ip" : { "type" : "string", "index":"not_analyzed"},
"country" : { "type" : "string", "index":"not_analyzed"}
}
}
})
7 nxapi/nxapi/nxtransform.py 100644 → 100755
@@ -252,7 +252,7 @@ def fancy_display(self, full_wl, scores, template=None):
output.append("#Rule ({0}) {1}\n".format(rid, self.core_msg.get(rid, 'Unknown ..')))
if self.cfg["output"]["verbosity"] >= 4:
output.append("#total hits {0}\n".format(full_wl['total_hits']))
for x in ["content", "peers", "uri", "var_name"]:
for x in ["content", "peers", "country", "uri", "var_name"]:
if x not in full_wl.keys():
continue
for y in full_wl[x]:
@@ -721,12 +721,15 @@ def gen_wl(self, tpl, rule={}):
if res['hits']['total'] > 0:
clist = []
peers = []
country = []
uri = []
var_name = []

for x in res['hits']['hits']:
if len(x.get("_source").get("ip", "")) > 0 and x.get("_source").get("ip", "") not in peers:
peers.append(x["_source"]["ip"])
if len(x.get("_source").get("country", "")) > 0 and x.get("_source").get("country", "") not in country:
country.append(x["_source"]["country"])
if len(x.get("_source").get("uri", "")) > 0 and x.get("_source").get("uri", "") not in uri:
uri.append(x["_source"]["uri"])
if len(x.get("_source").get("var_name", "")) > 0 and x.get("_source").get("var_name", "") not in var_name:
@@ -735,7 +738,7 @@ def gen_wl(self, tpl, rule={}):
clist.append(x["_source"]["content"])
if len(clist) >= 5:
break
retlist.append({'rule' : rule, 'content' : clist[:5], 'total_hits' : res['hits']['total'], 'peers' : peers[:5], 'uri' : uri[:5],
retlist.append({'rule' : rule, 'content' : clist[:5], 'total_hits' : res['hits']['total'], 'peers' : peers[:5], 'country' : country[:5], 'uri' : uri[:5],
'var_name' : var_name[:5]})
return retlist
return []
@@ -27,8 +27,8 @@

# Initialize logging
logging.basicConfig(stream=sys.stdout, level=logging.INFO,
format='%(asctime)s - %(levelname)s: %(message)s (%(name)s)',
datefmt='%c')
format=None,
datefmt=None)

def open_fifo(fifo):
try:
@@ -158,7 +158,7 @@ def get_filter(arg_filter):
use_ssl = bool(cfg.cfg["elastic"]["use_ssl"])
except KeyError:
use_ssl = False

es = elasticsearch.Elasticsearch(cfg.cfg["elastic"]["host"], use_ssl=use_ssl)
# Get ES version from the client and avail it at cfg
es_version = es.info()['version'].get('number', None)
@@ -182,7 +182,7 @@ def get_filter(arg_filter):
results = translate.full_auto()
if results:
for result in results:
logging.debug("{0}".format(result))
logging.info("{0}".format(result))
else:
logging.critical("No hits for this filter.")
sys.exit(1)
@@ -311,6 +311,13 @@ def get_filter(arg_filter):
logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3]))
except:
logging.warning("--malformed--")
logging.info(translate.red.format("# Top Country(ies) :"))
for e in translate.fetch_top(cfg.cfg["global_filters"], "country", limit=10):
try:
list_e = e.split()
logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3]))
except:
logging.warning("--malformed--")
sys.exit(0)


@@ -1,2 +1,2 @@
elasticsearch
elasticsearch==5.5.3
GeoIP
@@ -0,0 +1,4 @@
2019/04/17 11:37:19 [error] 11495#11495: *323360 NAXSI_EXLOG: ip=172.18.13.136&server=myexample.org&uri=%2F&id=1302&zone=ARGS&var_name=q&content=%3C%3E, client: 172.18.13.136, server: myexample.org, request: "GET /?q=%3C%3E HTTP/2.0", host: "myexample.org"
2019/04/17 11:37:19 [error] 11495#11495: *323360 NAXSI_FMT: ip=172.18.13.136&server=myexample.org&uri=/&learning=0&vers=0.56&total_processed=5&total_blocked=1&block=1&cscore0=$XSS&score0=8&zone0=ARGS&id0=1302&var_name0=q, client: 172.18.13.136, server: myexample.org, request: "GET /?q=%3C%3E HTTP/2.0", host: "myexample.org"
2019/04/17 11:37:46 [error] 11495#11495: *323360 NAXSI_EXLOG: ip=216.208.239.171&server=myexample.org&uri=%2F&id=1000&zone=ARGS&var_name=q&content=%22update%20table%20%28%29%22, client: 216.208.239.171, server: myexample.org, request: "GET /?q=%22update%20table%20()%22 HTTP/2.0", host: "myexample.org"
2019/04/17 11:37:46 [error] 11495#11495: *323360 NAXSI_FMT: ip=216.208.239.171&server=myexample.org&uri=/&learning=0&vers=0.56&total_processed=7&total_blocked=2&block=1&cscore0=$SQL&score0=8&zone0=ARGS&id0=1000&var_name0=q, client: 216.208.239.171, server: myexample.org, request: "GET /?q=%22update%20table%20()%22 HTTP/2.0", host: "myexample.org"

0 comments on commit d3cbf8a

Please sign in to comment.
You can’t perform that action at this time.