From 004fbf7c3574dd9f87fe4e6c71bb41f14472109e Mon Sep 17 00:00:00 2001 From: Cantoni Matteo Date: Mon, 26 Feb 2018 14:53:53 +0100 Subject: [PATCH 1/2] Improve virustotals table --- modules/python/dionaea/logsql.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/modules/python/dionaea/logsql.py b/modules/python/dionaea/logsql.py index 1f82e9eb..ef2b6841 100644 --- a/modules/python/dionaea/logsql.py +++ b/modules/python/dionaea/logsql.py @@ -289,6 +289,7 @@ def start(self): self.cursor.execute("""CREATE TABLE IF NOT EXISTS downloads ( + download_timestamp INTEGER NOT NULL, download INTEGER PRIMARY KEY, connection INTEGER, download_url TEXT, @@ -384,11 +385,13 @@ def start(self): self.cursor.execute("""CREATE INDEX IF NOT EXISTS mssql_commands_%s_idx ON mssql_commands (mssql_command_%s)""" % (idx, idx)) - - self.cursor.execute("""CREATE TABLE IF NOT EXISTS virustotals ( virustotal INTEGER PRIMARY KEY, virustotal_md5_hash TEXT NOT NULL, + virustotal_sha1_hash TEXT NOT NULL, + virustotal_sha256_hash TEXT NOT NULL, + virustotal_positives INTEGER NOT NULL, + virustotal_total INTEGER NOT NULL, virustotal_timestamp INTEGER NOT NULL, virustotal_permalink TEXT NOT NULL )""") @@ -797,11 +800,10 @@ def handle_incident_dionaea_download_complete_hash(self, icd): return attackid = self.attacks[con][1] logger.info("complete for attackid %i" % attackid) - self.cursor.execute("INSERT INTO downloads (connection, download_url, download_md5_hash) VALUES (?,?,?)", - (attackid, icd.url, icd.md5hash) ) + self.cursor.execute("INSERT INTO downloads (download_timestamp, connection, download_url, download_md5_hash) VALUES (?,?,?,?)", + (time.time(), attackid, icd.url, icd.md5hash) ) self.dbh.commit() - def handle_incident_dionaea_service_shell_listen(self, icd): con=icd.con if con not in self.attacks: @@ -874,9 +876,16 @@ def handle_incident_dionaea_modules_python_virustotal_report(self, icd): if j['response_code'] == 1: # file was known to virustotal permalink = j['permalink'] - date = j['scan_date'] - self.cursor.execute("INSERT INTO virustotals (virustotal_md5_hash, virustotal_permalink, virustotal_timestamp) VALUES (?,?,strftime('%s',?))", - (md5, permalink, date)) + scan_date = j['scan_date'] + sha1 = j['sha1'] + sha256 = j['sha256'] + positives = j['positives'] + total = j['total'] + + logger.debug("Trying to update table: virustotals (%s)", md5) + + self.cursor.execute("INSERT INTO virustotals (virustotal_md5_hash, virustotal_sha1_hash, virustotal_sha256_hash, virustotal_positives, virustotal_total, virustotal_permalink, virustotal_timestamp) VALUES (?,?,?,?,?,?,strftime('%s',?))", + (md5, sha1, sha256, positives, total, permalink, scan_date)) self.dbh.commit() virustotal = self.cursor.lastrowid From ab43f158419f8395212113cf155ec345c67338b3 Mon Sep 17 00:00:00 2001 From: Matteo Cantoni Date: Fri, 6 Apr 2018 16:15:51 +0200 Subject: [PATCH 2/2] use ALTER command to migrate existing tables --- modules/python/dionaea/logsql.py | 59 +++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/modules/python/dionaea/logsql.py b/modules/python/dionaea/logsql.py index ef2b6841..3730f964 100644 --- a/modules/python/dionaea/logsql.py +++ b/modules/python/dionaea/logsql.py @@ -210,10 +210,10 @@ def start(self): """UPDATE dcerpcserviceops SET dcerpcserviceop_name = 'NetPathCompare' WHERE dcerpcserviceop_name = 'NetCompare'""") logger.debug("... done") else: - logger.info("... not required") + logger.debug("... not required") except Exception as e: print(e) - logger.info("... not required") + logger.debug("... not required") self.cursor.execute("""CREATE TABLE IF NOT EXISTS emu_profiles ( @@ -229,10 +229,8 @@ def start(self): # 1) rename table, create the proper table try: logger.debug("Trying to update table: emu_services") - self.cursor.execute( - """SELECT emu_serivce FROM emu_services LIMIT 1""") - self.cursor.execute( - """ALTER TABLE emu_services RENAME TO emu_services_old""") + self.cursor.execute("""SELECT emu_serivce FROM emu_services LIMIT 1""") + self.cursor.execute("""ALTER TABLE emu_services RENAME TO emu_services_old""") update = True except Exception as e: logger.debug("... not required") @@ -261,7 +259,6 @@ def start(self): logger.debug( "Updating emu_services failed, copying old table failed (%s)" % e) - self.cursor.execute("""CREATE TABLE IF NOT EXISTS offers ( offer INTEGER PRIMARY KEY, @@ -273,20 +270,19 @@ def start(self): self.cursor.execute( """CREATE INDEX IF NOT EXISTS offers_url_idx ON offers (offer_url)""") - # fix a type on downloads table definition + # fix a typo on downloads table definition # downloads.downloads is wrong, should be downloads.download # 1) rename table, create the proper table try: - logger.debug("Trying to update table: downloads") + logger.debug("Trying to update table (fix typo): downloads") self.cursor.execute("""SELECT downloads FROM downloads LIMIT 1""") - self.cursor.execute( - """ALTER TABLE downloads RENAME TO downloads_old""") + self.cursor.execute("""ALTER TABLE downloads RENAME TO downloads_old""") update = True except Exception as e: - # print(e) + #print(e) logger.debug("... not required") update = False - + self.cursor.execute("""CREATE TABLE IF NOT EXISTS downloads ( download_timestamp INTEGER NOT NULL, @@ -296,8 +292,8 @@ def start(self): download_md5_hash TEXT -- CONSTRAINT downloads_connection_fkey FOREIGN KEY (connection) REFERENCES connections (connection) )""") - - # 2) copy all values to proper table, drop old table + + # 2) copy all values to proper table, drop old table try: if update == True: self.cursor.execute(""" @@ -316,6 +312,14 @@ def start(self): self.cursor.execute("""CREATE INDEX IF NOT EXISTS downloads_%s_idx ON downloads (download_%s)""" % (idx, idx)) + # 3) add new column 'download_timestamp' + try: + logger.debug("Trying to update table (add column): downloads") + self.cursor.execute("""SELECT download_timestamp FROM downloads LIMIT 1""") + logger.debug("... not required") + except Exception as e: + self.cursor.execute("""ALTER TABLE downloads ADD COLUMN download_timestamp INTEGER""") + logger.debug("... done") self.cursor.execute("""CREATE TABLE IF NOT EXISTS resolves ( @@ -384,7 +388,7 @@ def start(self): for idx in ["status"]: self.cursor.execute("""CREATE INDEX IF NOT EXISTS mssql_commands_%s_idx ON mssql_commands (mssql_command_%s)""" % (idx, idx)) - + self.cursor.execute("""CREATE TABLE IF NOT EXISTS virustotals ( virustotal INTEGER PRIMARY KEY, virustotal_md5_hash TEXT NOT NULL, @@ -396,9 +400,31 @@ def start(self): virustotal_permalink TEXT NOT NULL )""") + # add new columns about sha1, sha256 and positives/total + try: + logger.debug("Trying to update table: virustotals") + self.cursor.execute(""" + SELECT virustotal_sha1_hash,virustotal_sha256_hash,virustotal_positives,virustotal_total FROM virustotals LIMIT 1 + """) + logger.debug("... not required") + except Exception as e: + self.cursor.execute("""ALTER TABLE virustotals ADD COLUMN virustotal_sha1_hash TEXT""") + self.cursor.execute("""ALTER TABLE virustotals ADD COLUMN virustotal_sha256_hash TEXT""") + self.cursor.execute("""ALTER TABLE virustotals ADD COLUMN virustotal_positives INTEGER""") + self.cursor.execute("""ALTER TABLE virustotals ADD COLUMN virustotal_total INTEGER""") + logger.debug("... done") + for idx in ["md5_hash"]: self.cursor.execute("""CREATE INDEX IF NOT EXISTS virustotals_%s_idx ON virustotals (virustotal_%s)""" % (idx, idx)) + + for idx in ["sha1_hash"]: + self.cursor.execute("""CREATE INDEX IF NOT EXISTS virustotals_%s_idx + ON virustotals (virustotal_%s)""" % (idx, idx)) + + for idx in ["sha256_hash"]: + self.cursor.execute("""CREATE INDEX IF NOT EXISTS virustotals_%s_idx + ON virustotals (virustotal_%s)""" % (idx, idx)) self.cursor.execute("""CREATE TABLE IF NOT EXISTS virustotalscans ( virustotalscan INTEGER PRIMARY KEY, @@ -407,7 +433,6 @@ def start(self): virustotalscan_result TEXT )""") - for idx in ["scanner","result"]: self.cursor.execute("""CREATE INDEX IF NOT EXISTS virustotalscans_%s_idx ON virustotalscans (virustotalscan_%s)""" % (idx, idx))