Permalink
Browse files

Reformated files and reorganized imports

  • Loading branch information...
ali.ikinci@gmail.com
ali.ikinci@gmail.com committed Oct 29, 2011
1 parent 54bd244 commit 6f6a7bceb4fa95ba3f2e01dbb8c244d5c53a538b
Showing with 538 additions and 507 deletions.
  1. +15 −12 debug.py
  2. +17 −14 detection.py
  3. +13 −13 exampleImport.py
  4. +27 −25 gzip.py
  5. +32 −30 html.py
  6. +241 −231 jsunpackn.py
  7. +106 −96 pdf.py
  8. +35 −35 swf.py
  9. +52 −51 urlattr.py
View
@@ -1,8 +1,11 @@
#!/usr/bin/python
'''
Debug class - This is used by the Jsunpackn project to track performance statistics
Debug class - This is used by the Jsunpackn project to track performance
statistics
Within the application being debugged, you should not modify the members directly.
Instead, modify those elements by using the add* functions like addLaunch and addDetect
Within the application being debugged, you should not modify the members
directly. Instead, modify those elements by using the add* functions like
addLaunch and addDetect
'''
import time
@@ -23,33 +26,33 @@ def __init__(self, name, tmpdir):
def addLaunch(self,scriptFilename):
self.jsLaunches.append([{'filename':scriptFilename},self.endTimer()])
self.totalJsLaunches.append([{'filename':scriptFilename},self.endTimer()])
def addLaunch(self, scriptFilename):
self.jsLaunches.append([{'filename':scriptFilename}, self.endTimer()])
self.totalJsLaunches.append([{'filename':scriptFilename}, self.endTimer()])
if self.endTimer() > 3:
print 'addLaunch %s/%s took %.02f '% (self.name, scriptFilename, self.endTimer())
print 'addLaunch %s/%s took %.02f ' % (self.name, scriptFilename, self.endTimer())
def addDetect(self, contents):
self.ruleDetects.append([{'len':len(contents)}, self.endTimer()])
self.totalRuleDetects.append([{'len':len(contents)}, self.endTimer()])
fout = open('%s/signature_%03d' % (self.tmpdir,self.endTimer()),'wb')
fout = open('%s/signature_%03d' % (self.tmpdir, self.endTimer()), 'wb')
fout.write(contents)
fout.close()
def start_main(self):
self.before_decode = self.during_decode = time.time()
self.responsibility = { 'init': 0, 'decoding':0, 'shellcode':0 }
def record_main(self,type):
def record_main(self, type):
right_now = time.time()
self.responsibility[type] = right_now - self.during_decode
self.during_decode = right_now
def finalize_main(self):
if time.time() - self.before_decode > 3:
print 'main_decoder %s took %.02f seconds (ignored %d urls since last print)' % (self.name[0:20],time.time() - self.before_decode, DebugStats.ignored_main)
print 'main_decoder %s took %.02f seconds (ignored %d urls since last print)' % (self.name[0:20], time.time() - self.before_decode, DebugStats.ignored_main)
if self.responsibility['init'] > 0.5 or self.responsibility['shellcode'] > 1:
print '\t(%.02f init, %.02f decoding, %.02f sc)' % (self.responsibility['init'], self.responsibility['decoding'], self.responsibility['shellcode'])
print '\t(%.02f init, %.02f decoding, %.02f sc)' % (self.responsibility['init'], self.responsibility['decoding'], self.responsibility['shellcode'])
DebugStats.ignored_main = 0
elif DebugStats.ignored_main >= 10:
print 'main_decoder ignored %d urls since they are below threshold' % (DebugStats.ignored_main)
@@ -74,7 +77,7 @@ def jsTime(self):
def detectTime(self):
secs = 0
for datalen,elapsed in self.ruleDetects:
for datalen, elapsed in self.ruleDetects:
secs += elapsed
return secs
View
@@ -3,18 +3,21 @@
#Goal: use rules files to find content
#June 24, 2009
import sys,os,re,yara
import sys
import os
import re
import yara
class rules:
def __init__(self, ruleinput = '', respectLevel = True):
def __init__(self, ruleinput='', respectLevel=True):
'''ruleinput is a string that is the rule contents/data'''
#self.file = file
self.patterns = []
self.ruleinput = ruleinput
if len(ruleinput) <= 0:
if os.path.exists('rules'):
fin = open('rules','r')
fin = open('rules', 'r')
if fin:
self.ruleinput = fin.read()
fin.close()
@@ -25,9 +28,9 @@ def __init__(self, ruleinput = '', respectLevel = True):
def loadfile(self):
#previously this accepted filename as input, now we just use the rule input string
#self.rules = yara.compile(self.file)
self.rules = yara.compile(source = self.ruleinput)
self.rules = yara.compile(source=self.ruleinput)
def process(self, data, current_level = 0, from_pdf = True):
def process(self, data, current_level=0, from_pdf=True):
ret = [] #return value of all detections
matches = self.rules.match(data=data)
if matches:
@@ -76,31 +79,31 @@ def process(self, data, current_level = 0, from_pdf = True):
if alert:
ret.append([num,ref,msg,impact,rulemsg])
ret.append([num, ref, msg, impact, rulemsg])
return ret
def has_html(self, data):
#not used
return re.search('<script|<html|<body',data)
return re.search('<script|<html|<body', data)
def has_javascript(self,data):
data = re.sub('\0','',data)
def has_javascript(self, data):
data = re.sub('\0', '', data)
#if data.startswith('%PDF-') or data.startswith('%%PDF-'):
# #assume JavaScript to force parsing of PDF file
# return 1
#if data.startswith('CWS') or data.startswith('FWS'):
# #assume JavaScript to force parsing of SWF file
# return 1
return re.search('<script|document.write|eval\(|function |function\(|unescape[\(;]|var |=\s*[\'"][^\'"\\\n]+[\'"];',data, re.UNICODE)
return re.search('<script|document.write|eval\(|function |function\(|unescape[\(;]|var |=\s*[\'"][^\'"\\\n]+[\'"];', data, re.UNICODE)
if __name__ == '__main__':
rin = open('testrule','r')
rin = open('testrule', 'r')
rin_txt = rin.read()
rin.close()
r = rules(rin_txt,False) #Don't respect level because we aren't decoding
r = rules(rin_txt, False) #Don't respect level because we aren't decoding
for file in sys.argv[1:]:
fin = open(file, 'rb')
data = fin.read()
@@ -113,8 +116,8 @@ def has_javascript(self,data):
if impact > maximpact:
maximpact = impact
#txt += ' impact %d \t%s %s \t%s\n' % (impact,rulemsg,msg,ref)
txt += ' impact %d \t%s \t%s\n' % (impact,rulemsg,ref)
txt += ' impact %d \t%s \t%s\n' % (impact, rulemsg, ref)
txt += '\n'
print 'Detection %d file:%s\n%s' % (maximpact,file,txt)
print 'Detection %d file:%s\n%s' % (maximpact, file, txt)
View
@@ -19,9 +19,9 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
'''
import cgi,re,os
import datetime,time,socket
from hashlib import sha1
import datetime
import socket
import jsunpackn
# Error Reporting to /tmp
@@ -53,24 +53,24 @@ class cmdline_filler:
'currentproxy': '',
}
def __init__(self,inhash):
def __init__(self, inhash):
self.tmpdir = '/tmp' # these temporary files are necessary for decoding, but you can use any path and they will be deleted afterwards
self.logdir = self.outdir = '' # an empty storage filepath means no directory of output files will be created
self.decoded = '' #NO decoding logfile, otherwise = self.outdir + '/decoded.log'
for item in self.options:
setattr(self,item,self.options[item])
setattr(self, item, self.options[item])
#Feel free to hard code all your files "rules", "rules.ascii", and "htmlparse.config" in this file instead, only problem is updating them
fin = open('rules','r')
fin = open('rules', 'r')
if fin:
self.rules = fin.read()
fin.close()
fin = open('rules.ascii','r')
fin = open('rules.ascii', 'r')
if fin:
self.rulesAscii = fin.read()
fin.close()
if self.options['htmlparse']:
fin = open(self.options['htmlparse'],'r')
fin = open(self.options['htmlparse'], 'r')
self.htmlparseconfig = fin.read()
fin.close()
@@ -84,24 +84,24 @@ def main(userdata):
root_of_tree = '' # This can be empty but its sometimes useful to specify a filename here
url_or_name = '/' # This can also be empty but if you have the URL, you'd want to set that here
prevRooturl = {} # This can also be empty but if you want to decode something with more context its useful to keep state
js = jsunpackn.jsunpack(root_of_tree,[url_or_name,userdata,root_of_tree],options,prevRooturl)
js = jsunpackn.jsunpack(root_of_tree, [url_or_name, userdata, root_of_tree], options, prevRooturl)
for url in js.rooturl: # set all the state variables for printing
js.rooturl[url].seen = {}
results = ''
for url in [js.start]: #recursive
print 'The key %s has the following output in recursive mode' % (url)
results = js.rooturl[url].tostring('',True)[0] + '\n'
results = js.rooturl[url].tostring('', True)[0] + '\n'
print results
print 'Note that none of the files are actually created since self.outdir is empty.'
print 'Instead, you could go through each url and look at the decodings that it creates'
for url in js.rooturl:
print 'Looking at key %s, has %d files and %d messages, that follow:' % (url, len(js.rooturl[url].files), len(js.rooturl[url].msg))
for type,hash,data in js.rooturl[url].files:
print 'file type=%s, hash=%s, data=%d bytes' % (type,hash,len(data))
for printable,impact,msg in js.rooturl[url].msg:
print 'output message printable=%d, impact=%d, msg=%s' % (printable,impact,msg)
for type, hash, data in js.rooturl[url].files:
print 'file type=%s, hash=%s, data=%d bytes' % (type, hash, len(data))
for printable, impact, msg in js.rooturl[url].msg:
print 'output message printable=%d, impact=%d, msg=%s' % (printable, impact, msg)
if __name__ == "__main__":
main('eval("var a=123;");')
View
52 gzip.py
@@ -5,11 +5,13 @@
# based on Andrew Kuchling's minigzip.py distributed with the zlib module
import struct, sys, time
import zlib
import __builtin__
import struct
import sys
import time
import zlib
__all__ = ["GzipFile","open"]
__all__ = ["GzipFile", "open"]
FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
@@ -37,13 +39,13 @@ def write32u(output, value):
output.write(struct.pack("<L", value))
def read32(input):
i = input.read(4)
val = 0
if len(i) > 0:
#print 'cool with len %d' % (len(i))
val = struct.unpack("<l", i)[0]
return val
i = input.read(4)
val = 0
if len(i) > 0:
#print 'cool with len %d' % (len(i))
val = struct.unpack("<l", i)[0]
return val
def open(filename, mode="rb", compresslevel=9):
"""Shorthand for GzipFile(filename, mode, compresslevel).
@@ -121,7 +123,7 @@ def __init__(self, filename=None, mode=None,
self._init_write(filename)
self.compress = zlib.compressobj(compresslevel,
zlib.DEFLATED,
-zlib.MAX_WBITS,
- zlib.MAX_WBITS,
zlib.DEF_MEM_LEVEL,
0)
else:
@@ -170,10 +172,10 @@ def _read_gzip_header(self):
raise EOFError, 'end of file'
if magic != '\037\213':
raise IOError, 'Not a gzipped file'
method = ord( self.fileobj.read(1) )
method = ord(self.fileobj.read(1))
if method != 8:
raise IOError, 'Unknown compression method'
flag = ord( self.fileobj.read(1) )
flag = ord(self.fileobj.read(1))
# modtime = self.fileobj.read(4)
# extraflag = self.fileobj.read(1)
# os = self.fileobj.read(1)
@@ -182,25 +184,25 @@ def _read_gzip_header(self):
if flag & FEXTRA:
# Read & discard the extra field, if present
xlen = ord(self.fileobj.read(1))
xlen = xlen + 256*ord(self.fileobj.read(1))
xlen = xlen + 256 * ord(self.fileobj.read(1))
self.fileobj.read(xlen)
if flag & FNAME:
# Read and discard a null-terminated string containing the filename
while True:
s = self.fileobj.read(1)
if not s or s=='\000':
if not s or s == '\000':
break
if flag & FCOMMENT:
# Read and discard a null-terminated string containing a comment
while True:
s = self.fileobj.read(1)
if not s or s=='\000':
if not s or s == '\000':
break
if flag & FHCRC:
self.fileobj.read(2) # Read & discard the 16-bit header CRC
def write(self,data):
def write(self, data):
if self.mode != WRITE:
import errno
raise IOError(errno.EBADF, "write() on read-only GzipFile object")
@@ -210,10 +212,10 @@ def write(self,data):
if len(data) > 0:
self.size = self.size + len(data)
self.crc = zlib.crc32(data, self.crc)
self.fileobj.write( self.compress.compress(data) )
self.fileobj.write(self.compress.compress(data))
self.offset += len(data)
def read(self, size=-1):
def read(self, size= -1):
if self.mode != READ:
import errno
raise IOError(errno.EBADF, "read() on write-only GzipFile object")
@@ -265,7 +267,7 @@ def _read(self, size=1024):
if pos == self.fileobj.tell():
raise EOFError, "Reached EOF"
else:
self.fileobj.seek( pos ) # Return to original position
self.fileobj.seek(pos) # Return to original position
self._init_read()
self._read_gzip_header()
@@ -281,19 +283,19 @@ def _read(self, size=1024):
if buf == "":
uncompress = self.decompress.flush()
self._read_eof()
self._add_read_data( uncompress )
self._add_read_data(uncompress)
raise EOFError, 'Reached EOF'
uncompress = self.decompress.decompress(buf)
self._add_read_data( uncompress )
self._add_read_data(uncompress)
if self.decompress.unused_data != "":
# Ending case: we've come to the end of a member in the file,
# so seek back to the start of the unused data, finish up
# this member, and read a new gzip header.
# (The number of bytes to seek back is the length of the unused
# data, minus 8 because _read_eof() will rewind a further 8 bytes)
self.fileobj.seek( -len(self.decompress.unused_data)+8, 1)
self.fileobj.seek(-len(self.decompress.unused_data) + 8, 1)
# Check the CRC and file size, and set the flag so we read
# a new member on the next call
@@ -349,7 +351,7 @@ def __del__(self):
return
self.close()
def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
def flush(self, zlib_mode=zlib.Z_SYNC_FLUSH):
if self.mode == WRITE:
# Ensure the compressor's buffer is flushed
self.fileobj.write(self.compress.flush(zlib_mode))
@@ -397,7 +399,7 @@ def seek(self, offset):
self.read(1024)
self.read(count % 1024)
def readline(self, size=-1):
def readline(self, size= -1):
if size < 0:
size = sys.maxint
readsize = self.min_readsize
Oops, something went wrong.

0 comments on commit 6f6a7bc

Please sign in to comment.