Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve 'downloads' and 'virustotals' tables #153

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Improve 'downloads' and 'virustotals' tables #153

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
004fbf7
Improve virustotals table
mcantoni Feb 26, 2018
ab43f15
use ALTER command to migrate existing tables
mcantoni Apr 6, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
84 changes: 59 additions & 25 deletions modules/python/dionaea/logsql.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,10 +210,10 @@ def start(self):
"""UPDATE dcerpcserviceops SET dcerpcserviceop_name = 'NetPathCompare' WHERE dcerpcserviceop_name = 'NetCompare'""")
logger.debug("... done")
else:
logger.info("... not required")
logger.debug("... not required")
except Exception as e:
print(e)
logger.info("... not required")
logger.debug("... not required")

self.cursor.execute("""CREATE TABLE IF NOT EXISTS
emu_profiles (
Expand All @@ -229,10 +229,8 @@ def start(self):
# 1) rename table, create the proper table
try:
logger.debug("Trying to update table: emu_services")
self.cursor.execute(
"""SELECT emu_serivce FROM emu_services LIMIT 1""")
self.cursor.execute(
"""ALTER TABLE emu_services RENAME TO emu_services_old""")
self.cursor.execute("""SELECT emu_serivce FROM emu_services LIMIT 1""")
self.cursor.execute("""ALTER TABLE emu_services RENAME TO emu_services_old""")
update = True
except Exception as e:
logger.debug("... not required")
Expand Down Expand Up @@ -261,7 +259,6 @@ def start(self):
logger.debug(
"Updating emu_services failed, copying old table failed (%s)" % e)


self.cursor.execute("""CREATE TABLE IF NOT EXISTS
offers (
offer INTEGER PRIMARY KEY,
Expand All @@ -273,30 +270,30 @@ def start(self):
self.cursor.execute(
"""CREATE INDEX IF NOT EXISTS offers_url_idx ON offers (offer_url)""")

# fix a type on downloads table definition
# fix a typo on downloads table definition
# downloads.downloads is wrong, should be downloads.download
# 1) rename table, create the proper table
try:
logger.debug("Trying to update table: downloads")
logger.debug("Trying to update table (fix typo): downloads")
self.cursor.execute("""SELECT downloads FROM downloads LIMIT 1""")
self.cursor.execute(
"""ALTER TABLE downloads RENAME TO downloads_old""")
self.cursor.execute("""ALTER TABLE downloads RENAME TO downloads_old""")
update = True
except Exception as e:
# print(e)
#print(e)
logger.debug("... not required")
update = False

self.cursor.execute("""CREATE TABLE IF NOT EXISTS
downloads (
download_timestamp INTEGER NOT NULL,
download INTEGER PRIMARY KEY,
connection INTEGER,
download_url TEXT,
download_md5_hash TEXT
-- CONSTRAINT downloads_connection_fkey FOREIGN KEY (connection) REFERENCES connections (connection)
)""")

# 2) copy all values to proper table, drop old table
# 2) copy all values to proper table, drop old table
try:
if update == True:
self.cursor.execute("""
Expand All @@ -315,6 +312,14 @@ def start(self):
self.cursor.execute("""CREATE INDEX IF NOT EXISTS downloads_%s_idx
ON downloads (download_%s)""" % (idx, idx))

# 3) add new column 'download_timestamp'
try:
logger.debug("Trying to update table (add column): downloads")
self.cursor.execute("""SELECT download_timestamp FROM downloads LIMIT 1""")
logger.debug("... not required")
except Exception as e:
self.cursor.execute("""ALTER TABLE downloads ADD COLUMN download_timestamp INTEGER""")
logger.debug("... done")

self.cursor.execute("""CREATE TABLE IF NOT EXISTS
resolves (
Expand Down Expand Up @@ -383,19 +388,43 @@ def start(self):
for idx in ["status"]:
self.cursor.execute("""CREATE INDEX IF NOT EXISTS mssql_commands_%s_idx
ON mssql_commands (mssql_command_%s)""" % (idx, idx))




self.cursor.execute("""CREATE TABLE IF NOT EXISTS virustotals (
virustotal INTEGER PRIMARY KEY,
virustotal_md5_hash TEXT NOT NULL,
virustotal_sha1_hash TEXT NOT NULL,
virustotal_sha256_hash TEXT NOT NULL,
virustotal_positives INTEGER NOT NULL,
virustotal_total INTEGER NOT NULL,
virustotal_timestamp INTEGER NOT NULL,
virustotal_permalink TEXT NOT NULL
)""")

# add new columns about sha1, sha256 and positives/total
try:
logger.debug("Trying to update table: virustotals")
self.cursor.execute("""
SELECT virustotal_sha1_hash,virustotal_sha256_hash,virustotal_positives,virustotal_total FROM virustotals LIMIT 1
""")
logger.debug("... not required")
except Exception as e:
self.cursor.execute("""ALTER TABLE virustotals ADD COLUMN virustotal_sha1_hash TEXT""")
self.cursor.execute("""ALTER TABLE virustotals ADD COLUMN virustotal_sha256_hash TEXT""")
self.cursor.execute("""ALTER TABLE virustotals ADD COLUMN virustotal_positives INTEGER""")
self.cursor.execute("""ALTER TABLE virustotals ADD COLUMN virustotal_total INTEGER""")
logger.debug("... done")

for idx in ["md5_hash"]:
self.cursor.execute("""CREATE INDEX IF NOT EXISTS virustotals_%s_idx
ON virustotals (virustotal_%s)""" % (idx, idx))

for idx in ["sha1_hash"]:
self.cursor.execute("""CREATE INDEX IF NOT EXISTS virustotals_%s_idx
ON virustotals (virustotal_%s)""" % (idx, idx))

for idx in ["sha256_hash"]:
self.cursor.execute("""CREATE INDEX IF NOT EXISTS virustotals_%s_idx
ON virustotals (virustotal_%s)""" % (idx, idx))

self.cursor.execute("""CREATE TABLE IF NOT EXISTS virustotalscans (
virustotalscan INTEGER PRIMARY KEY,
Expand All @@ -404,7 +433,6 @@ def start(self):
virustotalscan_result TEXT
)""")


for idx in ["scanner","result"]:
self.cursor.execute("""CREATE INDEX IF NOT EXISTS virustotalscans_%s_idx
ON virustotalscans (virustotalscan_%s)""" % (idx, idx))
Expand Down Expand Up @@ -797,11 +825,10 @@ def handle_incident_dionaea_download_complete_hash(self, icd):
return
attackid = self.attacks[con][1]
logger.info("complete for attackid %i" % attackid)
self.cursor.execute("INSERT INTO downloads (connection, download_url, download_md5_hash) VALUES (?,?,?)",
(attackid, icd.url, icd.md5hash) )
self.cursor.execute("INSERT INTO downloads (download_timestamp, connection, download_url, download_md5_hash) VALUES (?,?,?,?)",
(time.time(), attackid, icd.url, icd.md5hash) )
self.dbh.commit()


def handle_incident_dionaea_service_shell_listen(self, icd):
con=icd.con
if con not in self.attacks:
Expand Down Expand Up @@ -874,9 +901,16 @@ def handle_incident_dionaea_modules_python_virustotal_report(self, icd):

if j['response_code'] == 1: # file was known to virustotal
permalink = j['permalink']
date = j['scan_date']
self.cursor.execute("INSERT INTO virustotals (virustotal_md5_hash, virustotal_permalink, virustotal_timestamp) VALUES (?,?,strftime('%s',?))",
(md5, permalink, date))
scan_date = j['scan_date']
sha1 = j['sha1']
sha256 = j['sha256']
positives = j['positives']
total = j['total']

logger.debug("Trying to update table: virustotals (%s)", md5)

self.cursor.execute("INSERT INTO virustotals (virustotal_md5_hash, virustotal_sha1_hash, virustotal_sha256_hash, virustotal_positives, virustotal_total, virustotal_permalink, virustotal_timestamp) VALUES (?,?,?,?,?,?,strftime('%s',?))",
(md5, sha1, sha256, positives, total, permalink, scan_date))
self.dbh.commit()

virustotal = self.cursor.lastrowid
Expand Down