Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix bug of creating wrong CSV outputs #66

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Fix bug of creating wrong CSV outputs #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d9dd162
Remove anyrun, since cfscrape library needs to fixed. With the latest…
anotherbridge May 16, 2023
dd9b824
Annotate that MD5 and SHA1 are not used for security, c.f. CWE-327.
anotherbridge May 16, 2023
b13924e
Add timeout in requests to avoid hanging indefinitely.
anotherbridge May 16, 2023
569f684
Add some compliance information.
anotherbridge May 16, 2023
2022c50
Fix broken CSV output due to delimiter characters in some fields.
anotherbridge May 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions lib/munin_csv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
#!/usr/bin/env python3
# coding: utf-8

import codecs
import traceback

Expand Down Expand Up @@ -54,24 +57,24 @@ def writeCSV(info, resultFile):
"""
try:
with codecs.open(resultFile, 'a', encoding='utf8') as fh_results:
fields = []
# Print every field from the field list to the output file
for field_pretty in CSV_FIELD_ORDER:
field = CSV_FIELDS[field_pretty]
try:
field = info[field]
except KeyError as e:
field = "False"
field = info.get(field, "False")
try:
field = str(field).replace(r'"', r'\"').replace("\n", " ")
except AttributeError as e:
traceback.print_exc()
fh_results.write("%s;" % field)
fields.append(field.replace(";", ","))
# Append vendor scan results
for vendor in VENDORS:
if vendor in info['vendor_results']:
fh_results.write("%s;" % info['vendor_results'][vendor])
fields.append(info['vendor_results'][vendor].replace(";", ","))
else:
fh_results.write("-;")
fields.append("-")

fh_results.write(";".join(fields))
fh_results.write('\n')
except:
traceback.print_exc()
Expand Down
85 changes: 31 additions & 54 deletions munin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
#!/usr/bin/env python3
# coding: utf-8

__AUTHOR__ = 'Florian Roth'
__VERSION__ = "0.22.0 January 2023"
__LICENSE__ = "Apache-2.0"

"""
Install dependencies with:
Expand Down Expand Up @@ -40,7 +42,6 @@
from lib.munin_csv import writeCSVHeader, writeCSV, CSV_FIELDS
import lib.connections as connections
from lib.munin_stdout import printResult, printHighlighted, printKeyLine
import cfscrape
# Handle modules that may be difficult to install
# e.g. pymisp has no Debian package, selenium is obsolete

Expand Down Expand Up @@ -208,9 +209,6 @@ def processLine(line, debug):
# URLhaus
uh_info = getURLhaus(info['md5'], info['sha256'])
info.update(uh_info)
# AnyRun
#ar_info = getAnyRun(info['sha256'])
#info.update(ar_info)
# CAPE
ca_info = getCAPE(info['md5'], info['sha1'], info['sha256'])
info.update(ca_info)
Expand Down Expand Up @@ -349,10 +347,11 @@ def getMalShareInfo(hash):
return info
try:
#print("Malshare URL: %s" % (MAL_SHARE_API % (MAL_SHARE_API_KEY, hash)))
response_query = requests.get(MAL_SHARE_API % (MAL_SHARE_API_KEY, hash),
timeout=15,
proxies=connections.PROXY,
headers=FAKE_HEADERS)
response_query = requests.get(
MAL_SHARE_API % (MAL_SHARE_API_KEY, hash),
timeout=15,
proxies=connections.PROXY,
headers=FAKE_HEADERS)
if args.debug:
print("[D] Querying Malshare: %s" % response_query.request.url)
#print(response_query.content)
Expand Down Expand Up @@ -438,13 +437,13 @@ def getIntezerInfo(sha256):
response.raise_for_status()
session = requests.session()
session.headers['Authorization'] = session.headers['Authorization'] = 'Bearer %s' % response.json()['result']

response = session.get(INTEZER_URL + '/files/{}'.format(hash))
if response.status_code == 404 or response.status_code == 410:
return info
else:
info['intezer_available'] = True

response.raise_for_status()
report = response.json()
if args.debug:
Expand Down Expand Up @@ -644,7 +643,10 @@ def getValhalla(sha256):
"sha256": sha256,
"apikey": VALHALLA_API_KEY,
}
response = requests.post(VALHALLA_URL, data=data, proxies=connections.PROXY)
response = requests.post(VALHALLA_URL,
data=data,
proxies=connections.PROXY,
timeout=15)
if args.debug:
print("[D] VALHALLA Response: '%s'" % response.json())
res = response.json()
Expand Down Expand Up @@ -677,7 +679,11 @@ def downloadHybridAnalysisSample(hash):
# Querying Hybrid Analysis
if args.debug:
print("[D] Requesting download of sample: %s" % preparedURL)
response = requests.get(preparedURL, params={'environmentId':'100'}, headers=headers, proxies=connections.PROXY)
response = requests.get(preparedURL,
params={'environmentId':'100'},
headers=headers,
proxies=connections.PROXY,
timeout=15)

# If the response is a json file
if response.headers["Content-Type"] == "application/json":
Expand Down Expand Up @@ -767,7 +773,9 @@ def getTotalHashInfo(sha1):
# Querying Hybrid Analysis
if args.debug:
print("[D] Querying Totalhash: %s" % preparedURL)
response = requests.get(preparedURL, proxies=connections.PROXY)
response = requests.get(preparedURL,
proxies=connections.PROXY,
timeout=15)
# print "Response: '%s'" % response.content
if response.content and \
'0 of 0 results' not in response.content and \
Expand Down Expand Up @@ -850,39 +858,6 @@ def getCAPE(md5, sha1, sha256):
return info


def getAnyRun(sha256):
"""
Retrieves information from AnyRun Service
:param sha256: hash value
:return info: info object
"""
info = {'anyrun_available': False}
if sha256 == "-":
return info
try:

if args.debug:
print("[D] Querying Anyrun")
cfscraper = cfscrape.create_scraper()
response = cfscraper.get(URL_ANYRUN % sha256, proxies=connections.PROXY)


if args.debug:
print("[D] Anyrun Response Code: %s" %response.status_code)

if response.status_code == 200:
info['anyrun_available'] = True
except ConnectionError as e:
print("Error while accessing AnyRun: connection failed")
if args.debug:
traceback.print_exc()
except Exception as e:
print("Error while accessing AnyRun")
if args.debug:
traceback.print_exc()
return info


def getVirusBayInfo(hash):
"""
Retrieves information from VirusBay https://beta.virusbay.io/
Expand All @@ -897,7 +872,9 @@ def getVirusBayInfo(hash):
preparedURL = "%s%s" % (VIRUSBAY_URL, hash)
if args.debug:
print("[D] Querying Virusbay: %s" % preparedURL)
response = requests.get(preparedURL, proxies=connections.PROXY).json()
response = requests.get(preparedURL,
proxies=connections.PROXY,
timeout=15).json()
# If response has the correct content
info['virusbay_available'] = False
#print(response)
Expand Down Expand Up @@ -1155,8 +1132,8 @@ def generateHashes(fileData):
"""
hashes = {'md5': '', 'sha1': '', 'sha256': ''}
try:
md5 = hashlib.md5()
sha1 = hashlib.sha1()
md5 = hashlib.md5(usedforsecurity=False)
sha1 = hashlib.sha1(usedforsecurity=False)
sha256 = hashlib.sha256()
md5.update(fileData)
sha1.update(fileData)
Expand Down Expand Up @@ -1250,7 +1227,7 @@ def showVTquota():
parser.add_argument('--cli', action='store_true', help='Run Munin in command line interface mode', default=False)
parser.add_argument('--rescan', action='store_true', help='Trigger a rescan of each analyzed file', default=False)
parser.add_argument('--debug', action='store_true', default=False, help='Debug output')

args = parser.parse_args()


Expand Down Expand Up @@ -1434,7 +1411,7 @@ def showVTquota():
except Exception as e:
traceback.print_exc()

# Query Valhalla for hashes matching the search word
# Query Valhalla for hashes matching the search word
if args.vh:
if not VALHALLA_API_KEY or VALHALLA_API_KEY == "-":
print("[E] Cannot query Valhalla without API Key")
Expand Down Expand Up @@ -1473,14 +1450,14 @@ def showVTquota():
print("Problems with converting timestamp %s" % timestamp_str)

if VH_RULE_CUTOFF:
# skip sample if
# - we already have it in cache or
# skip sample if
# - we already have it in cache or
# - it's too old for --vhmaxage
# - enough samples from this rule
if inCache(hashh) or \
now - vhmaxage > timestamp_hash or \
(
rule_count[rulename] and
rule_count[rulename] and
len(rule_count) / rule_count[rulename] > VH_RULE_CUTOFF and
# only skip after having 10+ samples of this rule to avoid problems on a fresh vt-hash-db.json
rule_count[rulename] > 10
Expand Down