Skip to content
This repository has been archived by the owner on Nov 8, 2023. It is now read-only.

Fixed issues to setup nxapi on ES5 and added country location on stats and generated whitelists #467

Merged
merged 5 commits into from May 7, 2019
Merged

Fixed issues to setup nxapi on ES5 and added country location on stats and generated whitelists #467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7e7b248
changing default elasticsearch version to 5 in order to work smoothly
fernandomariano Apr 17, 2019
f2a6292
added countries for stats reporting and generating whitelist process
fernandomariano Apr 17, 2019
2dd81f8
fixing fielddata id on elasticsearch
fernandomariano Apr 17, 2019
ad41411
changing logging to the previous behavior (print command) - no format…
fernandomariano Apr 17, 2019
d6eb85e
added sample nginx log file
fernandomariano Apr 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion nxapi/nxapi.json
View file
Open in desktop
Expand Up @@ -8,7 +8,7 @@
"doctype" : "events",
"default_ttl" : "7200",
"max_size" : "1000",
"version" : "2"
"version" : "5"
},
"syslogd": {
"host" : "0.0.0.0",
Expand Down
14 changes: 10 additions & 4 deletions nxapi/nxapi/nxparse.py
View file
Open in desktop
Expand Up @@ -428,12 +428,15 @@ def set_mappings(self):
# That is why time based indexes are recommended over this sort of thing and why
# _ttl was deprecated in the first place)
#"_ttl" : { "enabled" : "true", "default" : "4d" },
"properties" : { "var_name" : {"type": "keyword"},
"properties" : {
"id" : {"type": "keyword"},
"var_name" : {"type": "keyword"},
"uri" : {"type": "keyword"},
"zone" : {"type": "keyword"},
"server" : {"type": "keyword"},
"whitelisted" : {"type" : "keyword"},
"ip" : {"type" : "keyword"}
"ip" : {"type" : "keyword"},
"country" : {"type" : "keyword"}
}
}
})
Expand Down Expand Up @@ -464,13 +467,16 @@ def set_mappings(self):
body={
"events" : {
"_ttl" : { "enabled" : "true", "default" : "4d" },
"properties" : { "var_name" : {"type": "string", "index":"not_analyzed"},
"properties" : {
"id" : {"type": "string", "index":"not_analyzed"},
"var_name" : {"type": "string", "index":"not_analyzed"},
"uri" : {"type": "string", "index":"not_analyzed"},
"zone" : {"type": "string", "index":"not_analyzed"},
"server" : {"type": "string", "index":"not_analyzed"},
"whitelisted" : {"type" : "string", "index":"not_analyzed"},
"content" : {"type" : "string", "index":"not_analyzed"},
"ip" : { "type" : "string", "index":"not_analyzed"}
"ip" : { "type" : "string", "index":"not_analyzed"},
"country" : { "type" : "string", "index":"not_analyzed"}
}
}
})
Expand Down
7 changes: 5 additions & 2 deletions nxapi/nxapi/nxtransform.py 100644 → 100755
View file
Open in desktop
Expand Up @@ -252,7 +252,7 @@ def fancy_display(self, full_wl, scores, template=None):
output.append("#Rule ({0}) {1}\n".format(rid, self.core_msg.get(rid, 'Unknown ..')))
if self.cfg["output"]["verbosity"] >= 4:
output.append("#total hits {0}\n".format(full_wl['total_hits']))
for x in ["content", "peers", "uri", "var_name"]:
for x in ["content", "peers", "country", "uri", "var_name"]:
if x not in full_wl.keys():
continue
for y in full_wl[x]:
Expand Down Expand Up @@ -721,12 +721,15 @@ def gen_wl(self, tpl, rule={}):
if res['hits']['total'] > 0:
clist = []
peers = []
country = []
uri = []
var_name = []

for x in res['hits']['hits']:
if len(x.get("_source").get("ip", "")) > 0 and x.get("_source").get("ip", "") not in peers:
peers.append(x["_source"]["ip"])
if len(x.get("_source").get("country", "")) > 0 and x.get("_source").get("country", "") not in country:
country.append(x["_source"]["country"])
if len(x.get("_source").get("uri", "")) > 0 and x.get("_source").get("uri", "") not in uri:
uri.append(x["_source"]["uri"])
if len(x.get("_source").get("var_name", "")) > 0 and x.get("_source").get("var_name", "") not in var_name:
Expand All @@ -735,7 +738,7 @@ def gen_wl(self, tpl, rule={}):
clist.append(x["_source"]["content"])
if len(clist) >= 5:
break
retlist.append({'rule' : rule, 'content' : clist[:5], 'total_hits' : res['hits']['total'], 'peers' : peers[:5], 'uri' : uri[:5],
retlist.append({'rule' : rule, 'content' : clist[:5], 'total_hits' : res['hits']['total'], 'peers' : peers[:5], 'country' : country[:5], 'uri' : uri[:5],
'var_name' : var_name[:5]})
return retlist
return []
15 changes: 11 additions & 4 deletions nxapi/nxtool.py
View file
Open in desktop
Expand Up @@ -27,8 +27,8 @@

# Initialize logging
logging.basicConfig(stream=sys.stdout, level=logging.INFO,
format='%(asctime)s - %(levelname)s: %(message)s (%(name)s)',
datefmt='%c')
format=None,
datefmt=None)

def open_fifo(fifo):
try:
Expand Down Expand Up @@ -158,7 +158,7 @@ def get_filter(arg_filter):
use_ssl = bool(cfg.cfg["elastic"]["use_ssl"])
except KeyError:
use_ssl = False

es = elasticsearch.Elasticsearch(cfg.cfg["elastic"]["host"], use_ssl=use_ssl)
# Get ES version from the client and avail it at cfg
es_version = es.info()['version'].get('number', None)
Expand All @@ -182,7 +182,7 @@ def get_filter(arg_filter):
results = translate.full_auto()
if results:
for result in results:
logging.debug("{0}".format(result))
logging.info("{0}".format(result))
else:
logging.critical("No hits for this filter.")
sys.exit(1)
Expand Down Expand Up @@ -311,6 +311,13 @@ def get_filter(arg_filter):
logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3]))
except:
logging.warning("--malformed--")
logging.info(translate.red.format("# Top Country(ies) :"))
for e in translate.fetch_top(cfg.cfg["global_filters"], "country", limit=10):
try:
list_e = e.split()
logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3]))
except:
logging.warning("--malformed--")
sys.exit(0)


Expand Down
2 changes: 1 addition & 1 deletion nxapi/requirements.txt
View file
Open in desktop
@@ -1,2 +1,2 @@
elasticsearch
elasticsearch==5.5.3
GeoIP
4 changes: 4 additions & 0 deletions nxapi/sample/nginx.log
View file
Open in desktop
@@ -0,0 +1,4 @@
2019/04/17 11:37:19 [error] 11495#11495: *323360 NAXSI_EXLOG: ip=172.18.13.136&server=myexample.org&uri=%2F&id=1302&zone=ARGS&var_name=q&content=%3C%3E, client: 172.18.13.136, server: myexample.org, request: "GET /?q=%3C%3E HTTP/2.0", host: "myexample.org"
2019/04/17 11:37:19 [error] 11495#11495: *323360 NAXSI_FMT: ip=172.18.13.136&server=myexample.org&uri=/&learning=0&vers=0.56&total_processed=5&total_blocked=1&block=1&cscore0=$XSS&score0=8&zone0=ARGS&id0=1302&var_name0=q, client: 172.18.13.136, server: myexample.org, request: "GET /?q=%3C%3E HTTP/2.0", host: "myexample.org"
2019/04/17 11:37:46 [error] 11495#11495: *323360 NAXSI_EXLOG: ip=216.208.239.171&server=myexample.org&uri=%2F&id=1000&zone=ARGS&var_name=q&content=%22update%20table%20%28%29%22, client: 216.208.239.171, server: myexample.org, request: "GET /?q=%22update%20table%20()%22 HTTP/2.0", host: "myexample.org"
2019/04/17 11:37:46 [error] 11495#11495: *323360 NAXSI_FMT: ip=216.208.239.171&server=myexample.org&uri=/&learning=0&vers=0.56&total_processed=7&total_blocked=2&block=1&cscore0=$SQL&score0=8&zone0=ARGS&id0=1000&var_name0=q, client: 216.208.239.171, server: myexample.org, request: "GET /?q=%22update%20table%20()%22 HTTP/2.0", host: "myexample.org"