universal configuration add, iu update complete, etc. 13970116

README.md

@@ -1,2 +1,11 @@
 # iust_deep_fuzz
+## Bring Deep Neural Network for Use in Fuzz Testing
 A file format fuzzer base on deep neural networks.
+This repository is under a develop and its not documented well.
+If you downloaded it and have any question feel free to email me
+(*m-zakeri@live.com*) to get more information.
+---
+## About Version 0.1
+The main purpose of this version is to implement a free version of learn and fuzz paper 
+and improve the learn_&_fuzz algorithm. 
+---

binary_to_base64/base64_files/base64text.txt

@@ -0,0 +1 @@
+iVBORw0KGgoAAAANSUhEUgAAACAAAAAmBAMAAABXDYTaAAAAElBMVEX///////////8AAAD/AAD//wB+tBpvAAAAAXRSTlMAQObYZgAAAAFiS0dEAIgFHUgAAAAJcEhZcwAACxIAAAsSAdLdfvwAAACfSURBVCjPfdHBEcMgDETRTEpIBcnfDrRUYFRB+i8mB7BsOIQT82alQeLx5vX8cJ3HeVHIC7QelbDDzoSYEIDS9gotW/8HhLwCKL0C1gbIt4cBzIRmwidE4CUx3nHcSgzwBTlG4mwSBZ4bqX3Ith0qCHVoHapHy1B2aha1zMyMqB45gEpsYA842Epu4MyWvj5KY0LtwA7XLNjWaLHuFOAH7HBQQtXy4gMAAAAASUVORK5CYA==

binary_to_base64/base64encoding.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Nov  8 00:32:17 2017
+
+@author: Morteza
+"""
+
+import base64
+
+## part 1 convert binary to text using base64 encoding
+
+#read binary file
+with open("png1.png", "rb") as pdf_file:
+    ascii_byte_string = base64.b64encode(pdf_file.read())
+
+#print (ascii_byte_string)
+
+with open("base64text.txt", encoding='utf-8', mode='w') as text_file:
+    t = str(ascii_byte_string,'utf-8')
+    text_file.write(t)
+
+print("part1 done!")
+
+
+## part 2 convert text to binary using base64 decoding
+
+# read text file
+with open("base64text.txt", encoding='utf-8', mode='r') as text_file:
+    x = text_file.read()
+
+x = base64.b64decode(x)
+#print(x)
+with open("original_binary_file.png", "wb") as orginal_binary_file:
+    orginal_binary_file.write(x)
+
+print("part2 done!")

incremental_update/config.py → config.py

@@ -1,16 +1,33 @@
-config = {
+iu_config = {
     'single_object_update': False,  # [False, True]
     'portion_of_rewrite_objects': 1/3.,  # [1/4., 1/3., 1/2.]
     'update_policy': 'random',  # ['random', 'top-down', 'bottom_up']
-    'host1': './hosts/rawhost/host1.pdf',  # host1 path
-    'host2': './hosts/rawhost/host2.pdf',  # host2 path
-    'host3': './hosts/rawhost/host3.pdf',  # host3 path
-    'raw_host_directory': './hosts/rawhost/',  # raw hosts path
-    'new_host_directory': './iupdfs/',
-    'number_of_host': 3,
-    'iupdf_host1': './iupdfs/host1/',
-    'iupdf_host2': './iupdfs/host2/',
-    'iupdf_host3': './iupdfs/host3/',
+
+    # Old pdf file path (same hosts)
+    'number_of_hosts': 3,
+    'raw_host_directory': './hosts/rawhost/',  # raw hosts root directory path
+    'host1': './hosts/rawhost/host1.pdf',  # host1 full path
+    'host2': './hosts/rawhost/host2.pdf',  # host2 full path
+    'host3': './hosts/rawhost/host3.pdf',  # host3 full path
+    'host123': './hosts/rawhost/host123.pdf',  # host123 full path
+
+    # New generated/fuzzed objects path (by deep learning model)
+    'new_objects_path': 'not set yet',
+    'stream_directory_path': './dataset/pdfs/small_size_dataset/binary_streams/',
+
+    # New pdf files by attaching above new pdf objects
+    'new_pdfs_directory': './new_pdfs/',   # new generated pdf file root directory
+    'iupdf_host1': './new_pdfs/host1/',
+    'iupdf_host2': './new_pdfs/host2/',
+    'iupdf_host3': './new_pdfs/host3/',
+
+    # configuration setting to measure code coverage
+    'sut_dir': 'D:/afl/mupdf/platform/win32/Release/',
+    'sut_path': 'D:/afl/mupdf/platform/win32/Release/mutool.exe',
+    'sut_arguments': ' clean -difa ',
+
+    'visual_studio_developer_cmd_path':
+        'C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\Team Tools\\Performance Tools\\',
 }
 
 
@@ -49,5 +66,23 @@
 
 }
 
+
+pdf_objects_config = {
+
+}
+
+
+pdf_corpus_config = {
+    'corpus_root': 'D:/iust_pdf_corpus/corpus_garbage/',
+    'pdf_dir1_path': 'D:/iust_pdf_corpus/corpus_garbage/all_00_10kb/',  # 'Set 1 of IUST corpus' ==> 0-10 kb
+    'pdf_dir2_path': 'D:/iust_pdf_corpus/corpus_garbage/drive_deh_10_100kb/',  # 'Set 2 of IUST corpus' ==> 10-100 kb
+    'pdf_dir3_path': 'D:/iust_pdf_corpus/corpus_garbage/drive_h_100_900kb/',  # 'Set 3 of IUST corpus' ==> 100-900 kb
+    'pdf_dir4_path': 'D:/iust_pdf_corpus/corpus_garbage/mozilla/',  # 'Set 4 of IUST corpus' ==> mozilla corpus
+
+    'corpus_merged': 'D:/iust_pdf_corpus/corpus_merged/',
+    'corpus_merged_streams': 'D:/iust_pdf_corpus/corpus_merged_streams/'
+}
+
+
 # print(config['portion_of_rewrite_objects'])
 # print(learning_config['dataset_size'][2])

dataset/pdfs/small_size_dataset/binary_streams/pdftc_010k_0132.pdf009

@@ -0,0 +1,15 @@
+<?xpacket begin='' id='W5M0MpCehiHzreSzNTczkc9d'?>
+<?adobe-xap-filters esc="CRLF"?>
+<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='XMP toolkit 2.9.1-13, framework 1.6'>
+<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns:iX='http://ns.adobe.com/iX/1.0/'>
+<rdf:Description rdf:about='uuid:0b54026a-7757-11e2-0000-f9c513ab6907' xmlns:pdf='http://ns.adobe.com/pdf/1.3/' pdf:Producer='GPL Ghostscript 9.05'/>
+<rdf:Description rdf:about='uuid:0b54026a-7757-11e2-0000-f9c513ab6907' xmlns:xmp='http://ns.adobe.com/xap/1.0/'><xmp:ModifyDate>2013-02-12T11:04:10+01:00</xmp:ModifyDate>
+<xmp:CreateDate>2013-02-12T11:04:10+01:00</xmp:CreateDate>
+<xmp:CreatorTool>ChemDraw Ultra(tm) 11.0d1</xmp:CreatorTool></rdf:Description>
+<rdf:Description rdf:about='uuid:0b54026a-7757-11e2-0000-f9c513ab6907' xmlns:xapMM='http://ns.adobe.com/xap/1.0/mm/' xapMM:DocumentID='uuid:0b54026a-7757-11e2-0000-f9c513ab6907'/>
+<rdf:Description rdf:about='uuid:0b54026a-7757-11e2-0000-f9c513ab6907' xmlns:dc='http://purl.org/dc/elements/1.1/' dc:format='application/pdf'><dc:title><rdf:Alt><rdf:li xml:lang='x-default'>E-002.eps</rdf:li></rdf:Alt></dc:title></rdf:Description>
+</rdf:RDF>
+</x:xmpmeta>
+
+
+<?xpacket end='w'?>

dataset/pdfs/small_size_dataset/binary_streams/pdftc_010k_0140.pdf005

@@ -0,0 +1,5 @@
+x�m�KR�0D�>�O ��G:\!U�ff
ܿ���=���p�"�[R>2�d�y������+1qwk
N�
^�)�󖊐t�l҈M�#YQ�ܓ�N��&P#5\H�ՃZ�GZ 
+Ic
+'�A����:u��u犏�k�{҆Zo�Hm��\��H�����Ɨ����,���pO��Bmq���z`P��GC_Y�J���p%џ������c4
+o�"��Ed���'����2�'p��3j�G�+�tt�I�2L?%�26W#�Q5����ױ�
+��lpe����Hqj2����&�sP!�jP��V��#�g�1��^T��k�]���

dataset/pdfs/small_size_dataset/binary_streams/pdftc_010k_0208.pdf005

@@ -0,0 +1,2 @@
+x�M˱�0��_qcj���t!��ʆ�
+����E�?1ɀ<�z�[a����=%ڏ��M?Ŷ�J��uM	��SA!�T����g�5��s��9a�ְ����|UԲ��;�}@k9H�8�\"��ף��ۓ�Jb�h�/j)

dataset/pdfs/small_size_dataset/binary_streams/pdftc_010k_0218.pdf013

@@ -0,0 +1,15 @@
+<?xpacket begin='' id='W5M0MpCehiHzreSzNTczkc9d'?>
+<?adobe-xap-filters esc="CRLF"?>
+<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='XMP toolkit 2.9.1-13, framework 1.6'>
+<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns:iX='http://ns.adobe.com/iX/1.0/'>
+<rdf:Description rdf:about='c7419f2a-1fd3-11ec-0000-6abe4accfd7e' xmlns:pdf='http://ns.adobe.com/pdf/1.3/' pdf:Producer='GPL Ghostscript 9.02'/>
+<rdf:Description rdf:about='c7419f2a-1fd3-11ec-0000-6abe4accfd7e' xmlns:xmp='http://ns.adobe.com/xap/1.0/'><xmp:ModifyDate>2011-09-25T22:44:57+02:00</xmp:ModifyDate>
+<xmp:CreateDate>2011-09-25T22:44:57+02:00</xmp:CreateDate>
+<xmp:CreatorTool>UnknownApplication</xmp:CreatorTool></rdf:Description>
+<rdf:Description rdf:about='c7419f2a-1fd3-11ec-0000-6abe4accfd7e' xmlns:xapMM='http://ns.adobe.com/xap/1.0/mm/' xapMM:DocumentID='c7419f2a-1fd3-11ec-0000-6abe4accfd7e'/>
+<rdf:Description rdf:about='c7419f2a-1fd3-11ec-0000-6abe4accfd7e' xmlns:dc='http://purl.org/dc/elements/1.1/' dc:format='application/pdf'><dc:title><rdf:Alt><rdf:li xml:lang='x-default'>Untitled</rdf:li></rdf:Alt></dc:title></rdf:Description>
+</rdf:RDF>
+</x:xmpmeta>
+
+
+<?xpacket end='w'?>

dataset/pdfs/small_size_dataset/binary_streams/pdftc_010k_0219.pdf015

@@ -0,0 +1,15 @@
+<?xpacket begin='' id='W5M0MpCehiHzreSzNTczkc9d'?>
+<?adobe-xap-filters esc="CRLF"?>
+<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='XMP toolkit 2.9.1-13, framework 1.6'>
+<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns:iX='http://ns.adobe.com/iX/1.0/'>
+<rdf:Description rdf:about='8b3f228a-ce0f-11f0-0000-3545e7f6fcfd' xmlns:pdf='http://ns.adobe.com/pdf/1.3/' pdf:Producer='GPL Ghostscript 8.70'/>
+<rdf:Description rdf:about='8b3f228a-ce0f-11f0-0000-3545e7f6fcfd' xmlns:xmp='http://ns.adobe.com/xap/1.0/'><xmp:ModifyDate>2015-11-28T12:10:59-05:00</xmp:ModifyDate>
+<xmp:CreateDate>2015-11-28T12:10:59-05:00</xmp:CreateDate>
+<xmp:CreatorTool>groff version 1.18.1.4</xmp:CreatorTool></rdf:Description>
+<rdf:Description rdf:about='8b3f228a-ce0f-11f0-0000-3545e7f6fcfd' xmlns:xapMM='http://ns.adobe.com/xap/1.0/mm/' xapMM:DocumentID='8b3f228a-ce0f-11f0-0000-3545e7f6fcfd'/>
+<rdf:Description rdf:about='8b3f228a-ce0f-11f0-0000-3545e7f6fcfd' xmlns:dc='http://purl.org/dc/elements/1.1/' dc:format='application/pdf'><dc:title><rdf:Alt><rdf:li xml:lang='x-default'>Untitled</rdf:li></rdf:Alt></dc:title></rdf:Description>
+</rdf:RDF>
+</x:xmpmeta>
+
+
+<?xpacket end='w'?>

dataset/pdfs/small_size_dataset/binary_streams/pdftc_010k_0225.pdf015

@@ -0,0 +1,13 @@
+<?xpacket begin='' id='W5M0MpCehiHzreSzNTczkc9d'?>
+<?adobe-xap-filters esc="CRLF"?>
+<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='XMP toolkit 2.9.1-13, framework 1.6'>
+<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns:iX='http://ns.adobe.com/iX/1.0/'>
+<rdf:Description rdf:about='db19df6c-92e9-11e9-0000-15ab563b1a9a' xmlns:pdf='http://ns.adobe.com/pdf/1.3/' pdf:Producer='GPL Ghostscript 8.63'/>
+<rdf:Description rdf:about='db19df6c-92e9-11e9-0000-15ab563b1a9a' xmlns:xap='http://ns.adobe.com/xap/1.0/' xap:ModifyDate='2009-06-16T19:27:50-04:00' xap:CreateDate='2009-06-16T19:27:50-04:00'><xap:CreatorTool>groff version 1.18.1.4</xap:CreatorTool></rdf:Description>
+<rdf:Description rdf:about='db19df6c-92e9-11e9-0000-15ab563b1a9a' xmlns:xapMM='http://ns.adobe.com/xap/1.0/mm/' xapMM:DocumentID='db19df6c-92e9-11e9-0000-15ab563b1a9a'/>
+<rdf:Description rdf:about='db19df6c-92e9-11e9-0000-15ab563b1a9a' xmlns:dc='http://purl.org/dc/elements/1.1/' dc:format='application/pdf'><dc:title><rdf:Alt><rdf:li xml:lang='x-default'>Untitled</rdf:li></rdf:Alt></dc:title></rdf:Description>
+</rdf:RDF>
+</x:xmpmeta>
+
+
+<?xpacket end='w'?>

dataset/pdfs/small_size_dataset/binary_streams/pdftc_010k_0232.pdf015

@@ -0,0 +1,13 @@
+<?xpacket begin='' id='W5M0MpCehiHzreSzNTczkc9d'?>
+<?adobe-xap-filters esc="CRLF"?>
+<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='XMP toolkit 2.9.1-13, framework 1.6'>
+<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns:iX='http://ns.adobe.com/iX/1.0/'>
+<rdf:Description rdf:about='0bd3311a-15cf-11e9-0000-59bc4b81539c' xmlns:pdf='http://ns.adobe.com/pdf/1.3/' pdf:Producer='GPL Ghostscript 8.63'/>
+<rdf:Description rdf:about='0bd3311a-15cf-11e9-0000-59bc4b81539c' xmlns:xap='http://ns.adobe.com/xap/1.0/' xap:ModifyDate='2009-01-09T02:31:00+08:00' xap:CreateDate='2009-01-09T02:31:00+08:00'><xap:CreatorTool>dvips\(k\) 5.96 Copyright 2005 Radical Eye Software</xap:CreatorTool></rdf:Description>
+<rdf:Description rdf:about='0bd3311a-15cf-11e9-0000-59bc4b81539c' xmlns:xapMM='http://ns.adobe.com/xap/1.0/mm/' xapMM:DocumentID='0bd3311a-15cf-11e9-0000-59bc4b81539c'/>
+<rdf:Description rdf:about='0bd3311a-15cf-11e9-0000-59bc4b81539c' xmlns:dc='http://purl.org/dc/elements/1.1/' dc:format='application/pdf'><dc:title><rdf:Alt><rdf:li xml:lang='x-default'>6-12_.dvi</rdf:li></rdf:Alt></dc:title></rdf:Description>
+</rdf:RDF>
+</x:xmpmeta>
+
+
+<?xpacket end='w'?>

dataset/pdfs/small_size_dataset/binary_streams/pdftc_010k_0263.pdf013

@@ -0,0 +1,13 @@
+<?xpacket begin='' id='W5M0MpCehiHzreSzNTczkc9d'?>
+<?adobe-xap-filters esc="CRLF"?>
+<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='XMP toolkit 2.9.1-13, framework 1.6'>
+<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns:iX='http://ns.adobe.com/iX/1.0/'>
+<rdf:Description rdf:about='f303872d-0e7d-11e9-0000-fb99531c16d1' xmlns:pdf='http://ns.adobe.com/pdf/1.3/' pdf:Producer='GPL Ghostscript 8.63'/>
+<rdf:Description rdf:about='f303872d-0e7d-11e9-0000-fb99531c16d1' xmlns:xap='http://ns.adobe.com/xap/1.0/' xap:ModifyDate='2008-12-30T19:02:51+08:00' xap:CreateDate='2008-12-30T19:02:51+08:00'><xap:CreatorTool>dvips\(k\) 5.96 Copyright 2005 Radical Eye Software</xap:CreatorTool></rdf:Description>
+<rdf:Description rdf:about='f303872d-0e7d-11e9-0000-fb99531c16d1' xmlns:xapMM='http://ns.adobe.com/xap/1.0/mm/' xapMM:DocumentID='f303872d-0e7d-11e9-0000-fb99531c16d1'/>
+<rdf:Description rdf:about='f303872d-0e7d-11e9-0000-fb99531c16d1' xmlns:dc='http://purl.org/dc/elements/1.1/' dc:format='application/pdf'><dc:title><rdf:Alt><rdf:li xml:lang='x-default'>4-7_.dvi</rdf:li></rdf:Alt></dc:title></rdf:Description>
+</rdf:RDF>
+</x:xmpmeta>
+
+
+<?xpacket end='w'?>

dataset/pdfs/small_size_dataset/binary_streams/pdftc_010k_0285.pdf025

@@ -0,0 +1,38 @@
+<?xpacket begin="" id="W5M0MpCehiHzreSzNTczkc9d"?>
+<x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="Adobe XMP Core 5.4-c005 78.147326, 2012/08/23-13:03:03        ">
+   <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+      <rdf:Description rdf:about=""
+            xmlns:xmp="http://ns.adobe.com/xap/1.0/"
+            xmlns:dc="http://purl.org/dc/elements/1.1/"
+            xmlns:xmpMM="http://ns.adobe.com/xap/1.0/mm/">
+         <xmp:ModifyDate>2016-05-31T16:02:27+08:00</xmp:ModifyDate>
+         <xmp:CreateDate>2014-06-03T16:19:59+08:00</xmp:CreateDate>
+         <xmp:MetadataDate>2014-06-03T16:19:59+08:00</xmp:MetadataDate>
+         <dc:format>application/pdf</dc:format>
+         <xmpMM:DocumentID>uuid:33289722-6770-48e8-b7f2-75bfd84f4ca5</xmpMM:DocumentID>
+         <xmpMM:InstanceID>uuid:6d57ba73-82a8-4f6b-9d0d-11b564f59c17</xmpMM:InstanceID>
+      </rdf:Description>
+   </rdf:RDF>
+</x:xmpmeta>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<?xpacket end="w"?>