diff --git a/TODO b/TODO index eb38b08..d064343 100644 --- a/TODO +++ b/TODO @@ -9,9 +9,6 @@ [ENHANCEMENT] -- Add support for absorption of nodes leveraging the 'Bitfield collapse' - customization (i.e., absorption of bit-oriented nodes without a byte boundary). - (Counter-part of the generation supported feature.) - Add support for absorption of nodes whose existence has not been resolved yet. (Counter-part of the generation supported feature.) - Clean up test/test_integration.py diff --git a/data_models/.gitignore b/data_models/.gitignore index 4c793a0..41cabcf 100644 --- a/data_models/.gitignore +++ b/data_models/.gitignore @@ -4,6 +4,6 @@ !file_formats/*.py !protocols !protocols/*.py -!example*.py -!tuto*.py +!tutorial +!tutorial/*.py !__init__.py diff --git a/data_models/file_formats/jpg.py b/data_models/file_formats/jpg.py index d944538..acea045 100644 --- a/data_models/file_formats/jpg.py +++ b/data_models/file_formats/jpg.py @@ -49,7 +49,7 @@ class JPG_DataModel(DataModel): file_extension = 'jpg' name = 'jpg' - def absorb(self, data, idx): + def create_node_from_raw_data(self, data, idx, filename): nm = 'jpg_{:0>2d}'.format(idx) jpg = self.jpg.get_clone(nm, new_env=True) jpg.set_current_conf('ABS', recursive=True) @@ -68,7 +68,7 @@ def absorb(self, data, idx): print("--> Create {:s} from provided JPG sample [x:{:d}, y:{:d}].".format(nm, x, y)) return jpg else: - return Node(nm, values=['JPG ABSORBSION FAILED']) + return None def build_data_model(self): diff --git a/data_models/file_formats/pdf.py b/data_models/file_formats/pdf.py index abf7d9b..7b191a1 100644 --- a/data_models/file_formats/pdf.py +++ b/data_models/file_formats/pdf.py @@ -164,8 +164,8 @@ def get_number(name, int_m=0, int_M=2**40, dec_m=0, dec_M=2**20, enforce_unsigne e = Node(name) e.set_subnodes_with_csts([ 2, ['u>', [sign, 0, 1], [int_part, 0, 1], [end, 1]], - 3, ['u>', [sign, 0, 1], [int_part, 1], [end, 0, 1]], - 1, ['u>', [sign, 0, 1], [int_part, 1], [dot, 1]] + 3, ['u>', [sign, 0, 1], [int_part.get_clone('int_part_s2'), 1], [end.get_clone('float_part_s2'), 0, 1]], + 1, ['u>', [sign, 0, 1], [int_part.get_clone('int_part_s3'), 1], [dot, 1]] ]) e.set_semantics(NodeSemantics(['PDF_number', 'basic_type'])) diff --git a/data_models/file_formats/png.py b/data_models/file_formats/png.py index 8271b19..3f86c7b 100644 --- a/data_models/file_formats/png.py +++ b/data_models/file_formats/png.py @@ -31,7 +31,7 @@ class PNG_DataModel(DataModel): file_extension = 'png' name = 'png' - def absorb(self, data, idx): + def create_node_from_raw_data(self, data, idx, filename): nm = 'PNG_{:0>2d}'.format(idx) png = self.png.get_clone(nm, new_env=True) status, off, size, name = png.absorb(data, constraints=AbsNoCsts(size=True)) diff --git a/data_models/file_formats/zip.py b/data_models/file_formats/zip.py index 89f96bd..09114ba 100644 --- a/data_models/file_formats/zip.py +++ b/data_models/file_formats/zip.py @@ -33,7 +33,7 @@ class ZIP_DataModel(DataModel): file_extension = 'zip' name = 'zip' - def absorb(self, data, idx): + def create_node_from_raw_data(self, data, idx, filename): nm = 'ZIP_{:0>2d}'.format(idx) pkzip = self.pkzip.get_clone(nm, new_env=True) diff --git a/data_models/protocols/http.py b/data_models/protocols/http.py index 4233919..99deaeb 100755 --- a/data_models/protocols/http.py +++ b/data_models/protocols/http.py @@ -29,7 +29,7 @@ class HTTPModel(DataModel): name = 'HTTP' - def absorb(self, data, idx): + def create_node_from_raw_data(self, data, idx, filename): pass def build_data_model(self): diff --git a/data_models/protocols/pppoe.py b/data_models/protocols/pppoe.py index 25144a7..d750fdd 100644 --- a/data_models/protocols/pppoe.py +++ b/data_models/protocols/pppoe.py @@ -29,7 +29,7 @@ class PPPOE_DataModel(DataModel): file_extension = 'bin' - def absorb(self, data, idx): + def create_node_from_raw_data(self, data, idx, filename): pass def build_data_model(self): @@ -266,7 +266,7 @@ def cycle_tags(tag): pado = pppoe_msg.get_clone('pado') pado['.*/code'].set_values(value_type=UINT8(values=[0x7])) - pado['.*/code'].clear_attr(MH.Attr.Mutable) + # pado['.*/code'].clear_attr(MH.Attr.Mutable) padr = pppoe_msg.get_clone('padr') padr['.*/code'].set_values(value_type=UINT8(values=[0x19])) diff --git a/data_models/protocols/pppoe_strategy.py b/data_models/protocols/pppoe_strategy.py index d173644..ceed706 100644 --- a/data_models/protocols/pppoe_strategy.py +++ b/data_models/protocols/pppoe_strategy.py @@ -38,30 +38,37 @@ def retrieve_X_from_feedback(env, current_step, next_step, feedback, x='padi', u for source, status, timestamp, data in feedback: - msg_x = env.dm.get_atom(x) - msg_x.set_current_conf('ABS', recursive=True) - if x == 'padi': - mac_dst = b'\xff\xff\xff\xff\xff\xff' - elif x == 'padr': - if current_step.content is not None: - mac_src = current_step.content['.*/mac_src'] - env.mac_src = mac_src - else: - mac_src = env.mac_src - if mac_src is not None: - mac_dst = mac_src.to_bytes() - print('\n*** Destination MAC will be set to: {!r}'.format(mac_dst)) - else: - raise ValueError + if x == 'padi': + mac_dst = b'\xff\xff\xff\xff\xff\xff' + elif x == 'padr': + if current_step.content is not None: + mac_src = current_step.content['.*/mac_src'] + env.mac_src = mac_src + else: + mac_src = env.mac_src + if mac_src is not None: + mac_dst = mac_src.to_bytes() + # print('\n*** Destination MAC will be set to: {!r}'.format(mac_dst)) else: raise ValueError + else: + raise ValueError + + if data is None: + continue - if data is None: - continue - off = data.find(mac_dst) + off = -1 + while True: + off = data.find(mac_dst, off+1) + if off < 0: + break data = data[off:] + msg_x = env.dm.get_atom(x) + msg_x.set_current_conf('ABS', recursive=True) result = msg_x.absorb(data, constraints=AbsNoCsts(size=True, struct=True)) - print('\n [ ABS result: {!s} ]'.format(result)) + # print('\n [ ABS result: {!s} \n data: {!r} \n source: {!s} \ ts: {!s}]' + # .format(result, data, source, timestamp)) + if result[0] == AbsorbStatus.FullyAbsorbed: try: service_name = msg_x['.*/value/v101'].to_bytes() @@ -118,9 +125,10 @@ class t_fix_pppoe_msg_fields(Disruptor): def disrupt_data(self, dm, target, prev_data): n = prev_data.content + n.freeze() error_msg = '\n*** The node has no path to: {:s}. Thus, ignore it.\n'\ ' (probable reason: the node has been fuzzed in a way that makes the' \ - 'path unavailable)' + ' path unavailable)' if self.mac_src: try: n['.*/mac_dst'] = self.mac_src @@ -171,14 +179,14 @@ def disrupt_data(self, dm, target, prev_data): step_wait_padi = NoDataStep(fbk_timeout=10, fbk_mode=Target.FBK_WAIT_UNTIL_RECV, step_desc='Wait PADI') -dp_pado = DataProcess(process=[('ALT', None, UI(conf='fuzz')), - ('tTYPE', UI(init=1), UI(order=True, fuzz_mag=0.7)), +dp_pado = DataProcess(process=[('ALT', UI(conf='fuzz')), + ('tTYPE', UI(init=1, order=True, fuzz_mag=0.7)), 'FIX_FIELDS#pado1'], seed='pado') -dp_pado.append_new_process([('ALT', None, UI(conf='fuzz')), - ('tSTRUCT', UI(init=1), UI(deep=True)), 'FIX_FIELDS#pado2']) +dp_pado.append_new_process([('ALT', UI(conf='fuzz')), + ('tSTRUCT', UI(init=1, deep=True)), 'FIX_FIELDS#pado2']) step_send_pado = Step(dp_pado, fbk_timeout=0.1, fbk_mode=Target.FBK_WAIT_FULL_TIME) # step_send_pado = Step('pado') -step_end = Step(DataProcess(process=[('FIX_FIELDS#pado3', None, UI(reevaluate_csts=True))], +step_end = Step(DataProcess(process=[('FIX_FIELDS#pado3', UI(reevaluate_csts=True))], seed='padt'), fbk_timeout=0.1, fbk_mode=Target.FBK_WAIT_FULL_TIME) step_wait_padi.connect_to(step_send_pado, cbk_after_fbk=retrieve_padi_from_feedback_and_update) @@ -190,16 +198,16 @@ def disrupt_data(self, dm, target, prev_data): ### PADS fuzz scenario ### step_wait_padi = NoDataStep(fbk_timeout=10, fbk_mode=Target.FBK_WAIT_UNTIL_RECV, step_desc='Wait PADI') -step_send_valid_pado = Step(DataProcess(process=[('FIX_FIELDS#pads1', None, UI(reevaluate_csts=True))], +step_send_valid_pado = Step(DataProcess(process=[('FIX_FIELDS#pads1', UI(reevaluate_csts=True))], seed='pado'), fbk_timeout=0.1, fbk_mode=Target.FBK_WAIT_FULL_TIME) -step_send_padt = Step(DataProcess(process=[('FIX_FIELDS#pads2', None, UI(reevaluate_csts=True))], +step_send_padt = Step(DataProcess(process=[('FIX_FIELDS#pads2', UI(reevaluate_csts=True))], seed='padt'), fbk_timeout=0.1, fbk_mode=Target.FBK_WAIT_FULL_TIME) -dp_pads = DataProcess(process=[('ALT', None, UI(conf='fuzz')), - ('tTYPE#2', UI(init=1), UI(order=True, fuzz_mag=0.7)), +dp_pads = DataProcess(process=[('ALT', UI(conf='fuzz')), + ('tTYPE#2', UI(init=1, order=True, fuzz_mag=0.7)), 'FIX_FIELDS#pads3'], seed='pads') -dp_pads.append_new_process([('ALT', None, UI(conf='fuzz')), - ('tSTRUCT#2', UI(init=1), UI(deep=True)), 'FIX_FIELDS#pads4']) +dp_pads.append_new_process([('ALT', UI(conf='fuzz')), + ('tSTRUCT#2', UI(init=1, deep=True)), 'FIX_FIELDS#pads4']) step_send_fuzzed_pads = Step(dp_pads, fbk_timeout=0.1, fbk_mode=Target.FBK_WAIT_FULL_TIME) step_wait_padr = NoDataStep(fbk_timeout=10, fbk_mode=Target.FBK_WAIT_UNTIL_RECV, step_desc='Wait PADR/PADI') diff --git a/data_models/protocols/sms.py b/data_models/protocols/sms.py index 0d6bddd..01a9f7e 100644 --- a/data_models/protocols/sms.py +++ b/data_models/protocols/sms.py @@ -29,7 +29,7 @@ class SMS_DataModel(DataModel): file_extension = 'sms' - def absorb(self, data, idx): + def create_node_from_raw_data(self, data, idx, filename): pass def build_data_model(self): @@ -146,6 +146,12 @@ def build_data_model(self): {'name': 'TP-DCS', # Data Coding Scheme (refer to GSM 03.38) 'custo_set': MH.Custo.NTerm.CollapsePadding, 'contents': [ + {'name': 'msb', + 'determinist': True, + 'contents': BitField(subfield_sizes=[4], endian=VT.BigEndian, + subfield_values=[ + [0b1111,0b1101,0b1100,0b0000]], # last coding group + ) }, {'name': 'lsb1', 'determinist': True, 'exists_if': (BitFieldCondition(sf=0, val=[0b1111]), 'msb'), @@ -170,12 +176,6 @@ def build_data_model(self): [0b0000] # Default alphabet ] ) }, - {'name': 'msb', - 'determinist': True, - 'contents': BitField(subfield_sizes=[4], endian=VT.BigEndian, - subfield_values=[ - [0b1111,0b1101,0b1100,0b0000]], # last coding group - ) }, ]}, {'name': 'UDL', 'contents': LEN(vt=UINT8), diff --git a/data_models/tutorial/__init__.py b/data_models/tutorial/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data_models/example.py b/data_models/tutorial/example.py similarity index 98% rename from data_models/example.py rename to data_models/tutorial/example.py index 636b79b..55b3984 100644 --- a/data_models/example.py +++ b/data_models/tutorial/example.py @@ -176,12 +176,14 @@ def build_data_model(self): prefix.make_determinist() te3 = Node('EVT3') - te3.set_values(value_type=BitField(subfield_sizes=[4,4], subfield_values=[[0x5, 0x6], [0xF, 0xC]])) + te3.set_values(value_type=BitField(subfield_sizes=[4,4], endian=VT.LittleEndian, + subfield_values=[[0x5, 0x6], [0xF, 0xC]])) te3.set_fuzz_weight(8) # te3.make_determinist() te4 = Node('EVT4') - te4.set_values(value_type=BitField(subfield_sizes=[4,4], subfield_val_extremums=[[4, 8], [3, 15]])) + te4.set_values(value_type=BitField(subfield_sizes=[4,4], endian=VT.LittleEndian, + subfield_val_extremums=[[4, 8], [3, 15]])) te4.set_fuzz_weight(7) # te4.make_determinist() diff --git a/data_models/example_strategy.py b/data_models/tutorial/example_strategy.py similarity index 100% rename from data_models/example_strategy.py rename to data_models/tutorial/example_strategy.py diff --git a/data_models/tutorial/myproto.py b/data_models/tutorial/myproto.py new file mode 100644 index 0000000..e57b1c8 --- /dev/null +++ b/data_models/tutorial/myproto.py @@ -0,0 +1,131 @@ +################################################################################ +# +# Copyright 2018 Eric Lacombe +# +################################################################################ +# +# This file is part of fuddly. +# +# fuddly is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# fuddly is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with fuddly. If not, see +# +################################################################################ + +from framework.value_types import * +from framework.data_model import * +from framework.encoders import * + +class MyProto_DataModel(DataModel): + + name = 'myproto' + + def build_data_model(self): + + req_desc = \ + {'name': 'req', + 'contents': [ + {'name': 'header', + 'contents': BitField(subfield_sizes=[5,7,4], endian=VT.BigEndian, + subfield_values=[[0], [1,10,20], [1,2,3]], + subfield_descs=['reserved', 'cmd', 'version'])}, + + {'name': 'init', + 'exists_if': (BitFieldCondition(sf=1, val=[1]), 'header'), + 'contents': TIMESTAMP("%H:%M:%S"), + 'absorb_csts': AbsFullCsts(contents=False)}, + + {'name': 'register', + 'custo_clear': MH.Custo.NTerm.FrozenCopy, + 'exists_if': (BitFieldCondition(sf=1, val=10), 'header'), + 'contents': [ + {'name': 'payload', + 'contents': [ + {'name': 'file_qty', + 'contents': UINT16_be(min=2, max=8)}, + {'name': 'file_entry', + 'qty_from': 'file_qty', + 'contents': [ + {'name': 'filename', + 'contents': Filename(min_sz=1, max_sz=15, alphabet='abcdef')}, + {'name': 'len', + 'contents': UINT32_be()}, + {'name': 'content', + 'sync_size_with': 'len', + 'contents': String(min_sz=20, max_sz=50, alphabet='éùijklm:;!', + codec='latin-1')}, + {'name': 'crc32', + 'contents': CRC(vt=UINT32_be), + 'node_args': ['filename', 'content']}, + ]} + ]} + ]}, + + {'name': 'zregister', + 'exists_if/and': [(BitFieldCondition(sf=1, val=20), 'header'), + (BitFieldCondition(sf=2, val=3), 'header')], + 'encoder': GZIP_Enc(6), + 'contents': [ + {'name': 'zpayload', 'clone': 'payload'} + ]}, + + ]} + + req_atom = NodeBuilder(add_env=True).create_graph_from_desc(req_desc) + + init_atom = req_atom.get_clone('init', ignore_frozen_state=True) + init_atom['.*/header'].set_subfield(idx=1, val=1) + init_atom.unfreeze(recursive=True) + register_atom = req_atom.get_clone('register', ignore_frozen_state=True) + register_atom['.*/header'].set_subfield(idx=1, val=10) + register_atom.unfreeze(recursive=True) + zregister_atom = req_atom.get_clone('zregister', ignore_frozen_state=True) + zregister_atom['.*/header'].set_subfield(idx=1, val=20) + zregister_atom['.*/header'].set_subfield(idx=2, val=3) + zregister_atom.unfreeze(recursive=True) + + self.register(req_atom, init_atom, register_atom, zregister_atom) + + def validation_tests(self): + + data = [b'\x10 17:20:47', + + b'!@\x00\x02dffdecbcaab\x00\x00\x00/i\xe9!mkikl!jilmm!\xe9ml\xe9:;;\xe9\xe9' + b'\xe9kki\xf9!j\xf9j\xf9k\xf9::ji!!m:j:!\xcc\xc0\xc4\xedfab\x00\x00\x00*mmk!i;j' + b'\xf9\xe9\xe9!il;;m;\xe9!!l;ijklikl!kmlk\xf9!:;;jmmB\x8d8\x11', + + b'2\x80x\x9c%\x8e!\x0e\x02Q\x0cDW`\xf08\xd4\xc7\x82\xc1\xb6\xa7@\xa36\xbbl2\xd36{' + b',\x8e\xc0\x018\x03\x87\xa8D ' + b'\xea\xf8\x04;y\xf3\xf2\x86\xcd\xbcL\xf3\xb8\x0c\xc3p\xa0jc\x1aQ\xae\xa5\rQ\x91' + b'\x91"&\x92\xcd\xa3\xcf\xfb;\xfd\xb6\x8c\x1d>1\x85\x9a%\xca\xa4\x1b\n\xe6\xc5,' + b'U\x83\xabe\x13z\x98"\xcd\x9d\xd7\xe7\x87\xcb\xd4_\xbb\xce\xfd\xd4\xaaR\x9a\x99N' + b'\xc0\xd7\xed\xeb\xfd\x97\x9e\xa1&*A\xba$M`\x16@R\n\xbd\xa3\xc1\xa2\x05\xe1\x110' + b'\x96[\xb5\xba<\xd6\xe3\x17\x1c\xe3Sj '] + + ok = True + + for d in data: + atom = self.get_atom('req') + status, off, size, name = atom.absorb(d, constraints=AbsFullCsts()) + if status != AbsorbStatus.FullyAbsorbed: + ok = False + + print('Absorb Status: {!r}, {:d}, {:d}, {:s}'.format(status, off, size, name)) + print(' \_ length of original data: {:d}'.format(len(d))) + print(' \_ remaining: {!r}'.format(d[size:size+1000])) + + atom.show() + + return ok + + +data_model = MyProto_DataModel() \ No newline at end of file diff --git a/data_models/tutorial/myproto_strategy.py b/data_models/tutorial/myproto_strategy.py new file mode 100644 index 0000000..76537a4 --- /dev/null +++ b/data_models/tutorial/myproto_strategy.py @@ -0,0 +1,27 @@ +from framework.tactics_helpers import * +from framework.scenario import * + +tactics = Tactics() + + +def cbk_check_crc_error(env, current_step, next_step, fbk): + for source, status, timestamp, data in fbk: + if b'CRC error' in data: + return True + + +def set_init_v3(env, step): + step.content['.*/header'].set_subfield(2, 3) + + +init_step = Step('init', fbk_timeout=0.5, do_before_sending=set_init_v3) +v3cmd_step = Step('zregister', fbk_timeout=1) +final_step = FinalStep() + +init_step.connect_to(v3cmd_step) +v3cmd_step.connect_to(init_step, cbk_after_fbk=cbk_check_crc_error) +v3cmd_step.connect_to(final_step) + +sc_client_req = Scenario('basic', anchor=init_step) + +tactics.register_scenarios(sc_client_req) \ No newline at end of file diff --git a/data_models/tuto.py b/data_models/tutorial/tuto.py similarity index 83% rename from data_models/tuto.py rename to data_models/tutorial/tuto.py index bfb07c7..10eaf5a 100644 --- a/data_models/tuto.py +++ b/data_models/tutorial/tuto.py @@ -1,5 +1,4 @@ import sys -sys.path.append('.') from framework.plumbing import * @@ -8,13 +7,16 @@ from framework.data_model import * from framework.encoders import * import framework.dmhelpers.xml as xml +from framework.dmhelpers.json import * +from framework.dmhelpers.xml import tag_builder as xtb +from framework.dmhelpers.xml import xml_decl_builder class MyDF_DataModel(DataModel): file_extension = 'df' name = 'mydf' - def absorb(self, data, idx): + def create_node_from_raw_data(self, data, idx, filename): pass def build_data_model(self): @@ -449,6 +451,7 @@ def keycode_helper(blob, constraints, node_internals): {'name': 'inside_cpy', 'clone': 'i2'}, xml.tag_builder('D1', params={'p1':'a', 'p2': ['foo', 'bar'], 'p3': 'c'}, + specific_fuzzy_vals={'p2': ['myfuzzyvalue!']}, contents= \ {'name': 'inside', 'contents': [ @@ -461,11 +464,80 @@ def keycode_helper(blob, constraints, node_internals): 'contents': UINT16_be(values=[30,40,50])}, ] } + xml5_desc = \ + {'name': 'xml5', + 'contents': [ + xml_decl_builder(determinist=False), + xtb('command', params={'name': ['LOGIN', 'CMD_1', 'CMD_2']}, + nl_prefix=True, refs={'name': 'cmd_val'}, contents= \ + [xtb('LOGIN', condition=(RawCondition(val=['LOGIN']), 'cmd_val'), + params={'auth': ['cert', 'psk'], 'backend': ['ssh', 'serial']}, + specific_fuzzy_vals={'auth': ['None']}, determinist=False, contents= \ + [xtb('msg_id', contents=Node('mid', vt=INT_str(min=0))), + xtb('username', contents=['MyUser'], absorb_regexp='\w*'), + xtb('password', contents=['plopi'], + absorb_regexp='[^<\s]*')]), + xtb('CMD_1', condition=(RawCondition(val=['CMD_1']), 'cmd_val'), contents= \ + [{'name': 'msg_id'}, + xtb('counter', contents=Node('counter_val', vt=UINT8()))]), + xtb('CMD_2', condition=(RawCondition(val=['CMD_2']), 'cmd_val'), contents= \ + [{'name': 'msg_id'}, + {'name': 'counter'}, + xtb('filename', contents=Node('fln', vt=Filename(values=['/usr/bin/ls'])))]) + ]) + ]} + + json_sample_1 = \ + {"menu": { + "id": "file", + "value": "File", + "popup": { + "menuitem": [ + {"value": "New", "onclick": "CreateNewDoc()"}, + {"value": "Open", "onclick": "OpenDoc()"}, + {"value": "Close", "onclick": "CloseDoc()"} + ] + } + }} + + json1_desc = json_builder('json1', sample=json_sample_1) + + json_sample_2 = \ + {"glossary": { + "title": "example glossary", + "GlossDiv": { + "title": "S", + "GlossList": { + "GlossEntry": { + "ID": "SGML", + "SortAs": "SGML", + "GlossTerm": "Standard Generalized Markup Language", + "Acronym": "SGML", + "Abbrev": "ISO 8879:1986", + "GlossDef": { + "para": "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso": ["GML", "XML"] + }, + "GlossSee": "markup" + } + } + } + }} + + json2_desc = json_builder('json2', sample=json_sample_2) + + file_desc = \ + {'name': 'file', + 'contents': Filename(values=['test.txt']), + 'debug': True + } + self.register(test_node_desc, abstest_desc, abstest2_desc, separator_desc, sync_desc, len_gen_desc, misc_gen_desc, offset_gen_desc, shape_desc, for_network_tg1, for_network_tg2, for_net_default_tg, basic_intg, enc_desc, example_desc, - regex_desc, xml1_desc, xml2_desc, xml3_desc, xml4_desc) + regex_desc, xml1_desc, xml2_desc, xml3_desc, xml4_desc, xml5_desc, + json1_desc, json2_desc, file_desc) data_model = MyDF_DataModel() diff --git a/data_models/tuto_strategy.py b/data_models/tutorial/tuto_strategy.py similarity index 96% rename from data_models/tuto_strategy.py rename to data_models/tutorial/tuto_strategy.py index 95c6993..e551896 100644 --- a/data_models/tuto_strategy.py +++ b/data_models/tutorial/tuto_strategy.py @@ -14,12 +14,12 @@ def cbk_transition1(env, current_step, next_step, feedback): current_step.make_blocked() return False else: - print("\n\nFeedback received from {!s}. Let's go on".format(feedback.sources())) - for source, status, timestamp, data in feedback: + print("\n\nFeedback received from {!s}. Let's go on".format(feedback.sources_names())) + for source, status, timestamp, data in feedback: if data is not None: data = data[:15] print('*** Feedback entry:\n' - ' source: {:s}\n' + ' source: {!s}\n' ' status: {:d}\n' ' timestamp: {!s}\n' ' content: {!r} ...\n'.format(source, status, timestamp, data)) @@ -52,14 +52,14 @@ def before_data_processing_cbk(env, step): step.content.show() return True -periodic1 = Periodic(DataProcess(process=[('C', None, UI(nb=1)), 'tTYPE'], seed='enc'), +periodic1 = Periodic(DataProcess(process=[('C', UI(nb=1)), 'tTYPE'], seed='enc'), period=5) periodic2 = Periodic(Data('2nd Periodic (3s)\n'), period=3) ### SCENARIO 1 ### step1 = Step('exist_cond', fbk_timeout=1, set_periodic=[periodic1, periodic2], - do_before_sending=before_sending_cbk) -step2 = Step('separator', fbk_timeout=2, clear_periodic=[periodic1]) + do_before_sending=before_sending_cbk, vtg_ids=0) +step2 = Step('separator', fbk_timeout=2, clear_periodic=[periodic1], vtg_ids=1) empty = NoDataStep(clear_periodic=[periodic2]) step4 = Step('off_gen', fbk_timeout=0, step_desc='overriding the auto-description!') @@ -104,14 +104,14 @@ def before_data_processing_cbk(env, step): ### SCENARIO 4 & 5 ### dp = DataProcess(['tTYPE#NOREG'], seed='exist_cond', auto_regen=False) -dp.append_new_process([('tSTRUCT#NOREG', None, UI(deep=True))]) +dp.append_new_process([('tSTRUCT#NOREG', UI(deep=True))]) unique_step = Step(dp) unique_step.connect_to(unique_step) sc4 = Scenario('no_regen') sc4.set_anchor(unique_step) dp = DataProcess(['tTYPE#REG'], seed='exist_cond', auto_regen=True) -dp.append_new_process([('tSTRUCT#REG', None, UI(deep=True))]) +dp.append_new_process([('tSTRUCT#REG', UI(deep=True))]) unique_step = Step(dp) unique_step.connect_to(unique_step) sc5 = Scenario('auto_regen') diff --git a/docs/source/conf.py b/docs/source/conf.py index 324a9ec..18df4b8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -55,9 +55,9 @@ # built documents. # # The short X.Y version. -version = '0.26' +version = '0.27' # The full version, including alpha/beta/rc tags. -release = '0.26.0' +release = '0.27.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/data_model.rst b/docs/source/data_model.rst index 8f90a17..a0d9a67 100644 --- a/docs/source/data_model.rst +++ b/docs/source/data_model.rst @@ -149,10 +149,19 @@ custo_set, custo_clear customization mode. - ``MH.Custo.NTerm.CollapsePadding``: By default, this mode is *disabled*. - When enabled, every time two adjacent BitFields (within its scope) are found, they + When enabled, every time two adjacent ``BitField`` 's (within its scope) are found, they will be merged in order to remove any padding in between. This is done - "recursively" until any inner padding is removed. (Note this customization is currently - only supported for *generation* purpose and not for *absorption*.) + "recursively" until any inner padding is removed. + + .. note:: + To be compatible with an *absorption* operation, the non-terminal set with this + customization should comply with the following requirements: + + - It shall only contains ``BitField`` 's (which implies that no *separators* shall be used) + - The ``lsb_padding`` parameter shall be set to ``True`` on every related ``BitField`` 's. + - The ``endian`` parameter shall be set to ``VT.BigEndian`` on every related ``BitField`` 's. + - the ``qty`` keyword should not be used on the children except if it is equal to ``1``, + or ``(1,1)``. For *generator* node, the customizable behavior modes are: @@ -631,6 +640,9 @@ following parameters: to do it at the node level by using the data model keyword ``determinist`` (refer to :ref:`dm:node_prop_keywords`). +``values_desc`` [optional, default value: **None**] + Dictionary that maps integer values to their descriptions (character strings). Leveraged for + display purpose. Even if provided, all values do not need to be described. All these parameters are optional. If you don't specify all of them the constructor will let more freedom within the data model. But if @@ -657,6 +669,20 @@ corresponding outputs for a data generated from them: - :class:`framework.value_types.SINT64_le`: signed integer on 64 bit (2's complement), little endian - :class:`framework.value_types.INT_str`: ASCII encoded integer +For :class:`framework.value_types.INT_str`, additional parameters are available: + +``base`` [optional, default value: **10**] + Numerical base that have to be used to represent the integer into a string + +``letter_case`` [optional, default value: **'upper'**] + Only for hexadecimal base. It could be ``'upper'`` or ``'lower'`` for representing hexadecimal numbers + with these respective letter cases. + +``min_size`` [optional, default value: **None**] + If specified, the integer representation will have a minimum size (with added zeros when necessary). + +``reverse`` [optional, default value: **False**] + Reverse the order of the string if set to ``True``. String ------ diff --git a/docs/source/disruptors.rst b/docs/source/disruptors.rst index aa8cbcc..907c4d0 100644 --- a/docs/source/disruptors.rst +++ b/docs/source/disruptors.rst @@ -15,9 +15,19 @@ tTYPE - Advanced Alteration of Terminal Typed Node -------------------------------------------------- Description: - Perform alterations on typed nodes (one at a time) according to - its type and various complementary information (such as size, - allowed values, ...). + Perform alterations on typed nodes (one at a time) according to: + + - their type (e.g., INT, Strings, ...) + - their attributes (e.g., allowed values, minimum size, ...) + - knowledge retrieved from the data (e.g., if the input data uses separators, their symbols + are leveraged in the fuzzing) + - knowledge on the target retrieved from the project file or dynamically from feedback inspection + (e.g., C language, GNU/Linux OS, ...) + + If the input has different shapes (described in non-terminal nodes), this will be taken into + account by fuzzing every shape combinations. + + Note: this disruptor includes what tSEP does and goes beyond with respect to separators. Reference: :class:`framework.generic_data_makers.sd_fuzz_typed_nodes` @@ -25,7 +35,7 @@ Reference: Parameters: .. code-block:: none - generic args: + parameters: |_ init | | desc: make the model walker ignore all the steps until the provided | | one @@ -39,11 +49,9 @@ Parameters: | | default: -1 [type: int] |_ clone_node | | desc: if True the dmaker will always return a copy of the node. (for - | | stateless diruptors dealing with big data it can be usefull + | | stateless disruptors dealing with big data it can be useful | | to it to False) | | default: True [type: bool] - - specific args: |_ path | | desc: graph path regexp to select nodes on which the disruptor should | | apply @@ -53,7 +61,7 @@ Parameters: | | will reset its walk through the children nodes | | default: True [type: bool] |_ ign_sep - | | desc: when set to True, non-terminal separators will be ignored if + | | desc: when set to True, separators will be ignored if | | any are defined. | | default: False [type: bool] |_ fix @@ -73,16 +81,30 @@ Parameters: |_ fuzz_mag | | desc: order of magnitude for maximum size of some fuzzing test cases. | | default: 1.0 [type: float] + |_ determinism + | | desc: If set to 'True', the whole model will be fuzzed in a deterministic + | | way. Otherwise it will be guided by the data model determinism. + | | default: True [type: bool] + |_ leaf_determinism + | | desc: If set to 'True', each typed node will be fuzzed in a deterministic + | | way. Otherwise it will be guided by the data model determinism. + | | Note: this option is complementary to 'determinism' is it acts + | | on the typed node substitutions that occur through this disruptor + | | default: True [type: bool] + tSTRUCT - Alter Data Structure ------------------------------ Description: - For each node associated to existence constraints or quantity - constraints or size constraints, alter the constraint, one at a time, after each call - to this disruptor. If `deep` is set, enable new structure corruption cases, based on - the minimum and maximum amount of non-terminal nodes (within the - input data) specified in the data model. + Perform constraints alteration (one at a time) on each node that depends on another one + regarding its existence, its quantity, its size, ... + + If `deep` is set, enable more corruption cases on the data structure, based on the internals of + each non-terminal node: + + - the minimum and maximum amount of the subnodes of each non-terminal nodes + - ... Reference: :class:`framework.generic_data_makers.sd_struct_constraints` @@ -90,37 +112,35 @@ Reference: Parameters: .. code-block:: none - generic args: - |_ init - | | desc: make the model walker ignore all the steps until the provided - | | one - | | default: 1 [type: int] - |_ max_steps - | | desc: maximum number of steps (-1 means until the end) - | | default: -1 [type: int] - specific args: - |_ path - | | desc: graph path regexp to select nodes on which the disruptor should - | | apply - | | default: None [type: str] - |_ deep - | | desc: if True, enable corruption of minimum and maxium amount of non-terminal - | | nodes - | | default: False [type: bool] + parameters: + |_ init + | | desc: make the model walker ignore all the steps until the provided + | | one + | | default: 1 [type: int] + |_ max_steps + | | desc: maximum number of steps (-1 means until the end) + | | default: -1 [type: int] + |_ path + | | desc: graph path regexp to select nodes on which the disruptor should + | | apply + | | default: None [type: str] + |_ deep + | | desc: if True, enable corruption of non-terminal node internals + | | default: False [type: bool] Usage Example: A typical *disruptor chain* for leveraging this disruptor could be: .. code-block:: none - tWALK(path='path/to/some/node') tSTRUCT + tWALK(path='path/to/some/node') tSTRUCT .. note:: Test this chain with the data example found at :ref:`dm:pattern:existence-cond`, and set the path to the ``opcode`` node path. .. seealso:: Refer to :ref:`tuto:dmaker-chain` for insight - into *disruptor chains*. + into *disruptor chains*. @@ -137,10 +157,10 @@ Reference: Parameters: .. code-block:: none - generic args: + parameters: |_ clone_node | | desc: if True the dmaker will always return a copy of the node. (for - | | stateless diruptors dealing with big data it can be usefull + | | stateless disruptors dealing with big data it can be useful | | to it to False) | | default: True [type: bool] |_ init @@ -154,7 +174,6 @@ Parameters: | | desc: maximum number of test cases for a single node (-1 means until | | the end) | | default: -1 [type: int] - specific args: |_ conf | | desc: Change the configuration, with the one provided (by name), of | | all nodes reachable from the root, one-by-one. [default value @@ -177,37 +196,36 @@ Reference: Parameters: .. code-block:: none - generic args: - |_ clone_node - | | desc: if True the dmaker will always return a copy of the node. (for - | | stateless diruptors dealing with big data it can be usefull - | | to it to False) - | | default: True [type: bool] - |_ init - | | desc: make the model walker ignore all the steps until the provided - | | one - | | default: 1 [type: int] - |_ max_steps - | | desc: maximum number of steps (-1 means until the end) - | | default: -1 [type: int] - |_ runs_per_node - | | desc: maximum number of test cases for a single node (-1 means until - | | the end) - | | default: -1 [type: int] - specific args: - |_ path - | | desc: graph path regexp to select nodes on which the disruptor should - | | apply - | | default: None [type: str] - |_ order - | | desc: when set to True, the fuzzing order is strictly guided by the - | | data structure. Otherwise, fuzz weight (if specified in the - | | data model) is used for ordering - | | default: False [type: bool] - |_ deep - | | desc: when set to True, if a node structure has changed, the modelwalker - | | will reset its walk through the children nodes - | | default: True [type: bool] + parameters: + |_ clone_node + | | desc: if True the dmaker will always return a copy of the node. (for + | | stateless disruptors dealing with big data it can be useful + | | to it to False) + | | default: True [type: bool] + |_ init + | | desc: make the model walker ignore all the steps until the provided + | | one + | | default: 1 [type: int] + |_ max_steps + | | desc: maximum number of steps (-1 means until the end) + | | default: -1 [type: int] + |_ runs_per_node + | | desc: maximum number of test cases for a single node (-1 means until + | | the end) + | | default: -1 [type: int] + |_ path + | | desc: graph path regexp to select nodes on which the disruptor should + | | apply + | | default: None [type: str] + |_ order + | | desc: when set to True, the fuzzing order is strictly guided by the + | | data structure. Otherwise, fuzz weight (if specified in the + | | data model) is used for ordering + | | default: False [type: bool] + |_ deep + | | desc: when set to True, if a node structure has changed, the modelwalker + | | will reset its walk through the children nodes + | | default: True [type: bool] @@ -225,10 +243,10 @@ Reference: Parameters: .. code-block:: none - generic args: + parameters: |_ clone_node | | desc: if True the dmaker will always return a copy of the node. (for - | | stateless diruptors dealing with big data it can be usefull + | | stateless disruptors dealing with big data it can be useful | | to it to False) | | default: True [type: bool] |_ init @@ -242,7 +260,6 @@ Parameters: | | desc: maximum number of test cases for a single node (-1 means until | | the end) | | default: -1 [type: int] - specific args: |_ path | | desc: graph path regexp to select nodes on which the disruptor should | | apply @@ -263,6 +280,38 @@ Parameters: Stateless Disruptors ==================== +OP - Perform Operations on Nodes +-------------------------------- + +Description: + Perform an operation on the nodes specified by the regexp path. @op is an operation that + applies to a node and @params are a tuple containing the parameters that will be provided to + @op. If no path is provided, the root node will be used. + +Reference: + :class:`framework.generic_data_makers.d_operate_on_nodes` + +Parameters: + .. code-block:: none + + parameters: + |_ path + | | desc: Graph path regexp to select nodes on which the disruptor should + | | apply. + | | default: None [type: str] + |_ op + | | desc: The operation to perform on the selected nodes. + | | default: [type: method, function] + |_ params + | | desc: Tuple of parameters that will be provided to the operation. + | | (default: MH.Attr.Mutable) + | | default: (2,) [type: tuple] + |_ clone_node + | | desc: If True the dmaker will always return a copy of the node. (For + | | stateless disruptors dealing with big data it can be useful + | | to set it to False.) + | | default: False [type: bool] + MOD - Modify Node Contents -------------------------- @@ -279,22 +328,22 @@ Reference: Parameters: .. code-block:: none - specific args: - |_ path - | | desc: graph path regexp to select nodes on which the disruptor should - | | apply - | | default: None [type: str] - |_ clone_node - | | desc: if True the dmaker will always return a copy of the node. (for - | | stateless diruptors dealing with big data it can be usefull - | | to it to False) - | | default: False [type: bool] - |_ value - | | desc: the new value to inject within the data - | | default: '' [type: str] - |_ constraints - | | desc: constraints for the absorption of the new value - | | default: AbsNoCsts() [type: AbsCsts] + parameters: + |_ path + | | desc: graph path regexp to select nodes on which the disruptor should + | | apply + | | default: None [type: str] + |_ clone_node + | | desc: if True the dmaker will always return a copy of the node. (for + | | stateless disruptors dealing with big data it can be useful + | | to it to False) + | | default: False [type: bool] + |_ value + | | desc: the new value to inject within the data + | | default: '' [type: str] + |_ constraints + | | desc: constraints for the absorption of the new value + | | default: AbsNoCsts() [type: AbsCsts] @@ -313,19 +362,19 @@ Reference: Parameters: .. code-block:: none - specific args: - |_ path - | | desc: graph path regexp to select nodes on which the disruptor should - | | apply - | | default: None [type: str] - |_ clone_node - | | desc: if True the dmaker will always return a copy of the node. (for - | | stateless diruptors dealing with big data it can be usefull - | | to it to False) - | | default: False [type: bool] - |_ recursive - | | desc: apply the disruptor recursively - | | default: True [type: str] + parameters: + |_ path + | | desc: graph path regexp to select nodes on which the disruptor should + | | apply + | | default: None [type: str] + |_ clone_node + | | desc: if True the dmaker will always return a copy of the node. (for + | | stateless disruptors dealing with big data it can be useful + | | to it to False) + | | default: False [type: bool] + |_ recursive + | | desc: apply the disruptor recursively + | | default: True [type: str] @@ -340,7 +389,7 @@ Description: conditions*. .. seealso:: Refer to :ref:`dm:pattern:existence-cond` for insight - into existence conditions. + into existence conditions. Reference: :class:`framework.generic_data_makers.d_fix_constraints` @@ -348,16 +397,16 @@ Reference: Parameters: .. code-block:: none - specific args: - |_ path - | | desc: graph path regexp to select nodes on which the disruptor should - | | apply - | | default: None [type: str] - |_ clone_node - | | desc: if True the dmaker will always return a copy of the node. (for - | | stateless diruptors dealing with big data it can be usefull - | | to it to False) - | | default: False [type: bool] + parameters: + |_ path + | | desc: graph path regexp to select nodes on which the disruptor should + | | apply + | | default: None [type: str] + |_ clone_node + | | desc: if True the dmaker will always return a copy of the node. (for + | | stateless disruptors dealing with big data it can be useful + | | to it to False) + | | default: False [type: bool] ALT - Alternative Node Configuration @@ -372,20 +421,20 @@ Reference: Parameters: .. code-block:: none - specific args: - |_ path - | | desc: graph path regexp to select nodes on which the disruptor should - | | apply - | | default: None [type: str] - |_ recursive - | | desc: does the reachable nodes from the selected ones need also to - | | be changed? - | | default: True [type: bool] - |_ conf - | | desc: change the configuration, with the one provided (by name), of - | | all subnodes fetched by @path, one-by-one. [default value is - | | set dynamically with the first-found existing alternate configuration] - | | default: None [type: str] + parameters: + |_ path + | | desc: graph path regexp to select nodes on which the disruptor should + | | apply + | | default: None [type: str] + |_ recursive + | | desc: does the reachable nodes from the selected ones need also to + | | be changed? + | | default: True [type: bool] + |_ conf + | | desc: change the configuration, with the one provided (by name), of + | | all subnodes fetched by @path, one-by-one. [default value is + | | set dynamically with the first-found existing alternate configuration] + | | default: None [type: str] C - Node Corruption @@ -400,21 +449,21 @@ Reference: Parameters: .. code-block:: none - specific args: - |_ path - | | desc: graph path regexp to select nodes on which the disruptor should - | | apply - | | default: None [type: str] - |_ nb - | | desc: apply corruption on @nb Nodes fetched randomly within the data - | | model - | | default: 2 [type: int] - |_ ascii - | | desc: enforce all outputs to be ascii 7bits - | | default: False [type: bool] - |_ new_val - | | desc: if provided change the selected byte with the new one - | | default: None [type: str] + parameters: + |_ path + | | desc: graph path regexp to select nodes on which the disruptor should + | | apply + | | default: None [type: str] + |_ nb + | | desc: apply corruption on @nb Nodes fetched randomly within the data + | | model + | | default: 2 [type: int] + |_ ascii + | | desc: enforce all outputs to be ascii 7bits + | | default: False [type: bool] + |_ new_val + | | desc: if provided change the selected byte with the new one + | | default: None [type: str] Cp - Corruption at Specific Position @@ -429,16 +478,16 @@ Reference: Parameters: .. code-block:: none - specific args: - |_ new_val - | | desc: if provided change the selected byte with the new one - | | default: None [type: str] - |_ ascii - | | desc: enforce all outputs to be ascii 7bits - | | default: False [type: bool] - |_ idx - | | desc: byte index to be corrupted (from 1 to data length) - | | default: 1 [type: int] + parameters: + |_ new_val + | | desc: if provided change the selected byte with the new one + | | default: None [type: str] + |_ ascii + | | desc: enforce all outputs to be ascii 7bits + | | default: False [type: bool] + |_ idx + | | desc: byte index to be corrupted (from 1 to data length) + | | default: 1 [type: int] EXT - Make Use of an External Program @@ -453,18 +502,18 @@ Reference: Parameters: .. code-block:: none - specific args: - |_ path - | | desc: graph path regexp to select nodes on which the disruptor should - | | apply - | | default: None [type: str] - |_ cmd - | | desc: the command - | | default: None [type: list, tuple, str] - |_ file_mode - | | desc: if True the data will be provided through a file to the external - | | program, otherwise it will be provided on the command line directly - | | default: True [type: bool] + parameters: + |_ path + | | desc: graph path regexp to select nodes on which the disruptor should + | | apply + | | default: None [type: str] + |_ cmd + | | desc: the command + | | default: None [type: list, tuple, str] + |_ file_mode + | | desc: if True the data will be provided through a file to the external + | | program, otherwise it will be provided on the command line directly + | | default: True [type: bool] SIZE - Truncate @@ -479,14 +528,14 @@ Reference: Parameters: .. code-block:: none - specific args: - |_ sz - | | desc: truncate the data (or part of the data) to the provided size - | | default: 10 [type: int] - |_ path - | | desc: graph path regexp to select nodes on which the disruptor should - | | apply - | | default: None [type: str] + parameters: + |_ sz + | | desc: truncate the data (or part of the data) to the provided size + | | default: 10 [type: int] + |_ path + | | desc: graph path regexp to select nodes on which the disruptor should + | | apply + | | default: None [type: str] STRUCT - Shake Up Data Structure @@ -502,11 +551,11 @@ Reference: Parameters: .. code-block:: none - specific args: - |_ path - | | desc: graph path regexp to select nodes on which the disruptor should - | | apply - | | default: None [type: str] + parameters: + |_ path + | | desc: graph path regexp to select nodes on which the disruptor should + | | apply + | | default: None [type: str] diff --git a/docs/source/framework.rst b/docs/source/framework.rst index 80e99a7..89e1dd6 100644 --- a/docs/source/framework.rst +++ b/docs/source/framework.rst @@ -10,6 +10,17 @@ framework.basic_primitives module :undoc-members: :show-inheritance: +framework.data module +--------------------- + +.. automodule:: framework.data + :members: + :undoc-members: + :private-members: + :special-members: + :exclude-members: __dict__, __weakref__ + :show-inheritance: + framework.data_model module --------------------------- @@ -79,18 +90,6 @@ framework.target_helpers module :special-members: :exclude-members: __dict__, __weakref__ - -framework.targets module ------------------------- - -.. automodule:: framework.targets - :members: - :undoc-members: - :show-inheritance: - :private-members: - :special-members: - :exclude-members: __dict__, __weakref__ - framework.targets.network module -------------------------------- @@ -248,18 +247,6 @@ framework.scenario module :special-members: :exclude-members: __dict__, __weakref__ -framework.dmhelpers module --------------------------- - -.. automodule:: framework.dmhelpers - :members: - :undoc-members: - :show-inheritance: - :private-members: - :special-members: - :exclude-members: __dict__, __weakref__ - - framework.dmhelpers.generic module ---------------------------------- @@ -295,6 +282,39 @@ framework.evolutionary_helpers module :special-members: :exclude-members: __dict__, __weakref__ +framework.knowledge.feedback_collector module +--------------------------------------------- + +.. automodule:: framework.knowledge.feedback_collector + :members: + :undoc-members: + :show-inheritance: + :private-members: + :special-members: + :exclude-members: __dict__, __weakref__ + +framework.knowledge.feedback_handler module +------------------------------------------- + +.. automodule:: framework.knowledge.feedback_handler + :members: + :undoc-members: + :show-inheritance: + :private-members: + :special-members: + :exclude-members: __dict__, __weakref__ + +framework.knowledge.information module +-------------------------------------- + +.. automodule:: framework.knowledge.information + :members: + :undoc-members: + :show-inheritance: + :private-members: + :special-members: + :exclude-members: __dict__, __weakref__ + .. framework.plumbing module ----------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index fc35197..28fff9b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -22,6 +22,8 @@ Contents: scenario + knowledge + evolutionary_fuzzing disruptors diff --git a/docs/source/knowledge.rst b/docs/source/knowledge.rst new file mode 100644 index 0000000..2594806 --- /dev/null +++ b/docs/source/knowledge.rst @@ -0,0 +1,193 @@ +.. _knowledge-infra: + +Knowledge Infrastructure +************************ + +The *Knowledge Infrastructure* enables to: + +- to dynamically collect feedback from Targets (:ref:`targets`) and Probes + (:ref:`probes`), and extract information from it through dedicated handlers in order to + create knowledge (refer to :ref:`kn:handle-fbk`); + +- to add knowledge about the targets under test (e.g., the kind of OS, the used programming language, + the hardware, and so on) in your project file (refer to :ref:`kn:adding`); + +- and to leverage this knowledge in relevant fuddly subsystems or in user-defined scenarios, + disruptors, ... (refer to :ref:`kn:leverage`). For instance, fuzzing a :class:`framework.value_types.Filename` + typed-node with the disruptor tTYPE will adapt the generated data relative to the OS, Language, + and so on if this information is available. + + +.. _kn:get_knowledge: + +Get Knowledge about the Targets Under Test +------------------------------------------ + +.. _kn:handle-fbk: + +Defining a Feedback Handler to Create Knowledge from Targets' and Probes' Feedback +================================================================================== + +The :class:`knwoledge.feedback_handler.FeedbackHandler` class provides the frame to create knowledge based on +feedback retrieved by fuddly (essentially from targets themselves and the probes that +monitor them). + +In your projects, you can use already defined feedback handlers in order to automatically extract information +from feedback and create knowledge that will be directly usable in various relevant fuddly components +(refer to :ref:`kn:leverage`). + +Let's illustrate that with the ``tuto`` project (refer to ``/projects/tuto_proj.py``) that +register a ``fuddly``-defined feedback handler whose sole purpose is to present the feature: + +.. code-block:: python + :linenos: + + project.register_feedback_handler(TestFbkHandler()) + +This handler is defined as follows: + +.. code-block:: python + :linenos: + + class TestFbkHandler(FeedbackHandler): + + def extract_info_from_feedback(self, source, timestamp, content, status): + + if content is None: + return None + elif b'Linux' in content: + return OS.Linux + elif b'Windows' in content: + return OS.Windows + + +It implements the method :meth:`knowledge.feedback_handler.FeedbackHandler.extract_info_from_feedback` that is +called each time feedback are retrieved from a target or a probe with parameters enabling you to process it, +and return information about the target (in this case either :const:`OS.Linux` or :const:`OS.Windows`) +if it can or ``None`` if it is not able. + +The information concept is implemented through the class :class:`framework.knowledge.information.Info`, +and provide specific methods to increase +or decrease the confidence that we have about a specific information. Each time a feedback handler return +specific information like ``OS.Linux`` for instance, the framework would increase the confidence it has on it +through the method :meth:`framework.knowledge.information.Info.increase_trust`. Note that at any +given time you can look at the current confidence level for any information by using the +:meth:`framework.knowledge.information.Info.show_trust` method. + +The accumulation of information and the computed confidence level for each piece of it make up the +knowledge on the targets under test. + +If you want to look at the current state of the knowledge pool, you can issue the following +command from the FmkShell:: + + >> show_knowledge + +That will provide something similar to the following output:: + + -=[ Status of Knowledge ]=- + + Info: Language.C [TrustLevel.Maximum --> value: 50] + Info: Hardware.X86_64 [TrustLevel.Maximum --> value: 50] + +As dealing with feedback can be specific to your projects, you can obviously define +your own feedback handlers for matching your specific needs. In order to do that you will have to +create a new class that inherits from :class:`knwoledge.feedback_handler.FeedbackHandler` +and implements your specific behaviors. Then you will only need to register it in your :class:`framework.project.Project` +in order for its methods to be called automatically by fuddly at the relevant times. + +.. note:: + Even if initial purpose of feedback handlers is to create knowledge from retrieved information, it can be + used to trigger other kinds of actions that fit your needs. + +:class:`knowledge.feedback_handler.FeedbackHandler` provides other methods that could be useful to overload +to extract more information about the context of the feedback. Indeed, the method +:meth:`knowledge.feedback_handler.FeedbackHandler.notify_data_sending` is called each time data is sent +and provide you with useful contextual information: + +- the sent data; +- the date of emission; +- the targets. + + +.. _kn:adding: + +Adding Knowledge About the Targets Under Test in the Project File +================================================================= + +As seen in Section :ref:`kn:handle-fbk`, knowledge on the targets under test can be built upon +the information extracted from feedback retrieved while interacting with the targets. But it can also +be something known from the beginning. If you know you are dealing with a C program, and that program +is executed on an x86 architecture, then you would like to provide this knowledge right ahead, so +that fuddly could leverage them to optimize its fuzzing for instance. + +In order to provide such knowledge, you simply have to call :meth:`framework.project.Project.add_knowledge` +in your project file with your knowledge on the targets. + +.. code-block:: python + :linenos: + + project.add_knowledge( + Hardware.X86_64, + Language.C + ) + +Information Categories and How to Define More +============================================= + +The current information categories are: + +- :class:`framework.knowledge.information.OS` +- :class:`framework.knowledge.information.Hardware` +- :class:`framework.knowledge.information.Language` +- :class:`framework.knowledge.information.InputHandling` + +Depending on your project, you may want to define new specific information categories. In such case, +You will simply have to define new python enumeration that inherits from +:class:`framework.knowledge.information.Info` in your project file. Then, you would need to use them +in specific feedback handler (refer to :ref:`kn:handle-fbk`) in order to leverage them within +specific scenarios or disruptors for instance. + +.. _kn:leverage: + +Leveraging the Knowledge +------------------------ + +Automatic Fuddly Adaptation to Knowledge +======================================== + +**It is a work in progress.** + +Currently, data models that use the following node types +within their description will benefit from knowledge about the targets under test: + +- :class:`framework.value_types.String`: Specific cases related to :class:`framework.knowledge.information.Language` + are added. +- :class:`framework.value_types.Filename`: Specific cases related to + :class:`framework.knowledge.information.OS` and :class:`framework.knowledge.information.Language` + are added. + +If knowledge on the targets are provided to the framework (either from the project file or because +some in-use feedback handlers populated at some point the knowledge pool) then the previous type nodes +will restrict their own fuzzing cases, impacting directly the disruptor ``tTYPE`` (refer to :ref:`dis:ttype`) +in order to avoid doing irrelevant tests (e.g., providing a C format strings input to an ADA program). + +If there is no knowledge on a specific category, then all specific fuzzing cases related to that category +will still be provided. + + +Leveraging Knowledge in User-defined Components +=============================================== + +Knowledge on the targets under tests can be used by various components of the framework and is made +available to the user in various context like: + +- Scenario specification (refer to :ref:`scenario-infra`) where all callbacks can access the knowledge pool through the scenario environment + (:class:`framework.scenario.ScenarioEnv`) under the attribute `knowledge_source`. + +- Disruptors or generators implementation (refer to :ref:`tuto:disruptors`), through the attribute + :attr:`framework.tactics_helpers.DataMaker.knowledge_source`. + +- Data model description (refer to :ref:`data-model`), through the attribute + :attr:`framework.data_model.DataModel.knowledge_source`. + +These parameters refer to a global object defined for the project as a set of :class:`framework.knowledge.information.Info`. diff --git a/docs/source/probes.rst b/docs/source/probes.rst index aa6f3f8..a240fba 100644 --- a/docs/source/probes.rst +++ b/docs/source/probes.rst @@ -10,6 +10,9 @@ and providing the expected parameters. Besides, you have to provide them with a access the monitored system, namely a :class:`framework.monitor.Backend`. Note that you can use the same backend for simultaneous probes. +.. seealso:: + To define your own probe refer to :ref:`tuto:probes`. + Let's illustrate this with the following example where two probes are used to monitor a process through an SSH connection. One is used to check if the PID of the process has changed after each data sending, and the other one to check if the memory used by the process has exceeded @@ -87,8 +90,8 @@ Reference: :class:`framework.monitor.ProbePID` Description: - This generic probe enables you to monitor a process PID through an - SSH connection. + This generic probe enables you to monitor any modification of a process PID, + by specifying its name through the parameter ``process_name``. ProbeMem -------- diff --git a/docs/source/scenario.rst b/docs/source/scenario.rst index 7061b81..b75efe1 100644 --- a/docs/source/scenario.rst +++ b/docs/source/scenario.rst @@ -41,7 +41,7 @@ A First Example Let's begin with a simple example that interconnect 3 steps in a loop without any callback. .. note:: All the examples (or similar ones) of this chapter are provided in the file - ``/data_models/tuto_strategy.py``. + ``/data_models/tutorial/tuto_strategy.py``. .. code-block:: python :linenos: @@ -68,11 +68,21 @@ Let's begin with a simple example that interconnect 3 steps in a loop without an tactics.register_scenarios(sc1) -You should first note that scenarios have to be described in a ``*_strategy.py`` file that matches -the data model you base your scenarios on. In our case we use the data model ``mydf`` defined in -``tuto.py`` (refer to :ref:`dm:mydf` for further explanation on file organization). -The special object ``tactics`` (line 4) is usually used to register the data makers (`disruptors` or -`generators`) specific to a data model (refer to :ref:`tuto:disruptors` for details). It is also used +Note that scenarios can be described: + +- either in a ``*_strategy.py`` file that matches the data model you base your scenarios on; +- or in a project file, if your scenarios use different data models involved in your project or + the scenarios are agnostic to any data model but have a meaning only at project level. + +In what follows we illustrate how to describe scenarios in the context of the data model ``mydf`` defined in +``tuto.py`` (refer to :ref:`dm:mydf` for further explanation on file organization). Describing scenarios +in the context of a project will be the same except for the scenarios registration step, where the method +:meth:`framework.project.Project.register_scenarios` will have to be used to make them +available within the framework. + +In our example, the registration goes through the special object ``tactics`` (line 4) of ``tuto_strategy.py`` +which is usually used to register the data makers (`disruptors` or +`generators`) specific to a data model (refer to :ref:`tuto:disruptors` for details), but also used to register scenarios as shown in line 20. From line 9 to 11 we define 3 :class:`framework.scenario.Step`: @@ -153,10 +163,15 @@ Steps ----- The main objective of a :class:`framework.scenario.Step` is to command the generation and sending -of one or multiple data to the target selected in the framework. The data generation depends on +of one or multiple data to targets selected in the framework. The data generation depends on what has been provided to the parameter ``data_desc`` of a :class:`framework.scenario.Step`. This is described in the section :ref:`sc:dataprocess`. +Note that the data generated in one step will be sent by default to the first loaded target. If the +scenario you describe involve different targets, you could then refer to them by specifying virtual +target IDs in the step constructor thanks to the parameter ``vtg_ids``. Virtual target IDs are then +to be mapped to real targets within the project file. Refer to :ref:`multi-target-scenario`. + A step can also modify the way the feedback is handled after the data have been emitted by the framework. The parameters ``fbk_timeout``, and ``fbk_mode`` (refer to :ref:`targets`) are used for such purpose and are applied to the current target (by the framework) when the step is reached. @@ -265,16 +280,16 @@ A brief explanation is provided below: This type of callback takes the additional parameter ``feedback`` filled by the framework with the target and/or probes feedback further to the current step data sending. It is an object - :class:`framework.database.FeedbackHandler` that provides the handful method - :meth:`framework.database.FeedbackHandler.iter_entries` which returns a generator that iterates + :class:`framework.database.FeedbackGate` that provides the handful method + :meth:`framework.database.FeedbackGate.iter_entries` which returns a generator that iterates over: - all the feedback entries associated to a specific feedback ``source`` provided as a parameter---and for each entry the triplet ``(status, timestamp, content)`` is provided; - all the feedback entries if the ``source`` parameter is ``None``---and for each entry the 4-uplet ``(source, status, timestamp, content)`` is provided. Note that for such kind of iteration, the - :class:`framework.database.FeedbackHandler` object can also be directly used as - an iterator---avoiding a call to :meth:`framework.database.FeedbackHandler.iter_entries`. + :class:`framework.database.FeedbackGate` object can also be directly used as + an iterator---avoiding a call to :meth:`framework.database.FeedbackGate.iter_entries`. This object can also be tested as a boolean object, returning False if there is no feedback at all. @@ -335,7 +350,7 @@ service for instance. This is illustrated in the following example in the lines step1 = Step('exist_cond', fbk_timeout=2, set_periodic=[periodic1, periodic2]) step2 = Step('separator', fbk_timeout=5) step3 = NoDataStep() - step4 = Step(DataProcess(process=[('C',None,UI(nb=1)),'tTYPE'], seed='enc')) + step4 = Step(DataProcess(process=[('C', UI(nb=1)),'tTYPE'], seed='enc')) step1.connect_to(step2) step2.connect_to(step3, cbk_after_fbk=feedback_callback) @@ -404,7 +419,7 @@ The execution of this scenario will follow the pattern:: .. _sc:dataprocess: -Data generation process +Data Generation Process ----------------------- The data produced by a :class:`framework.scenario.Step` or a :class:`framework.scenario.Periodic` @@ -451,7 +466,7 @@ Here under examples of steps leveraging the different ways to describe their dat Step( Data('A raw message') ) - Step( DataProcess(process=['ZIP', 'tSTRUCT', ('SIZE', None, UI(sz=100))]) ) + Step( DataProcess(process=['ZIP', 'tSTRUCT', ('SIZE', UI(sz=100))]) ) Step( DataProcess(process=['C', 'tTYPE'], seed='enc') ) Step( DataProcess(process=['C'], seed=Data('my seed')) ) @@ -461,6 +476,32 @@ meaning the framework will be ordered to use :meth:`framework.target.Target.send a list of `data descriptors` (instead of one). +.. _multi-target-scenario: + +Scenario Involving Multiple Targets +----------------------------------- + +If you want to define a scenario that involves multiple targets, you will have to refer to the +different targets through virtual target IDs. +To illustrate such case, let's look at the ``ex1`` scenario defined in the ``tuto`` +data model (refer to the file ``data_models/tutorial/tuto_strategy.py``). ``step1`` and ``step2`` are defined with +respectively the virtual target ID ``0`` and the virtual target ID ``1``:: + + step1 = Step(... vtg_ids=0) + step2 = Step(... vtg_ids=1) + +Then, in order to use this scenario in your project you will have to provide a mapping with real targets +thanks to the method :meth:`framework.project.Project.map_targets_to_scenario`. For instance in the +``tuto`` project (refer to the file ``projects/tuto_proj.py``), a mapping is created for the +scenario ``ex1``:: + + project.map_targets_to_scenario('ex1', {0: 8, 1: 9, None: 9}) + +A mapping is a simple python dictionnary that maps virtual target IDs to real target IDs. In our +case, virtual IDs 0 and 1 have been mapped respectiveley to real IDs 8 and 9. Finally, the last +association with the ``None`` virtual target ID is to cover data generated by steps that did not +specify any virtual IDs at all. + .. _scenario-fuzz: Scenario Fuzzing @@ -504,7 +545,7 @@ an imaginary protocol. :scale: 100% .. note:: - It is described by the following code snippet extracted from ``data_models/tuto_strategy.py``: + It is described by the following code snippet extracted from ``data_models/tutorial/tuto_strategy.py``: .. code-block:: python :linenos: diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 2d87342..dc1e299 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -5,7 +5,7 @@ In this tutorial we will begin with the basic UI of ``fuddly``. Then we will see how to use ``fuddly`` directly from an advanced python interpreter like ``ipython``. Finally, we will walk through basic steps to create a new data model and the way to define specific -disruptors, +disruptors. Using ``fuddly`` simple UI: ``Fuddly Shell`` @@ -69,7 +69,7 @@ Note that ``fuddly`` looks for *Data Model* files (within ``data_models/``) and *Project* files (within ``projects/``) during its initialization. A *Project* file is used to describe the targets that can be tested, the logger behaviour, and optionally specific -monitoring means as well as some virtual operators. +monitoring means as well as some scenarios and/or virtual operators. .. seealso:: To create a new project file, and to describe the associated components refer to :ref:`tuto:project`. @@ -98,7 +98,7 @@ You can look at the defined targets by issuing the following command: -=[ Available Targets ]=- - [0] EmptyTarget + [0] EmptyTarget [ID: 307144] [1] LocalTarget [Program: display] [2] LocalTarget [Program: okular] [3] LocalTarget [Program: unzip, Args: -d ~/fuddly_data/workspace/] @@ -114,8 +114,15 @@ experiment without a real target. But let's say you want to fuzz the :linenos: :emphasize-lines: 1 - >> set_target 3 + >> load_targets 3 +.. note:: + You can also load several targets at the same time if you want to sequence different actions + through various systems or on the same system but through different kinds of interfaces + (represented by different targets). To do it, provide a list of target IDs to the + ``load_targets`` command. For instance to load the targets 1, 4 and 5, issue the command:: + + >> load_targets 1 4 5 .. seealso:: In order to define new targets, look at :ref:`targets-def`. @@ -168,7 +175,7 @@ issuing the following command: *** Data Model 'zip' loaded *** *** Logger is started *** - *** Target initialization *** + *** Target initialization: (0) EmptyTarget [ID: 307144] *** *** Monitor is started *** *** [ Fuzz delay = 0 ] *** @@ -190,14 +197,24 @@ issuing the following command: - The timeout value for checking target's health. (Can be changed through the command ``set_health_timeout``) +Finally, you may prefer to directly launch your project thanks to +the command ``run_project``. Indeed, by using it, you will automatically trigger the commands we +just talked about. Regarding the loaded data models it will initially load what is defined as default +in the project file. In the case of the ``standard`` project, if you issue the following command:: -Finally, note that if you know the target from the project file you -want to interact with, you can directly launch your project thanks to -the command ``run_project``. Basically by issuing ``run_project -standard 1``, you will automatically trigger the commands we just -talked about. Note this command will initially load the default data -model defined in the ``standard`` project file, which is the imaginary -data model used by our tutorial (``mydf``). +>> run_project standard + +the imaginary data model used by our tutorial (``mydf``) will be loaded and the default target +will be chosen, namely the ``EmptyTarget`` (usefull for testing purpose) with the ID 0. + +In order to run the project with the ``unzip`` target (ID 4), you will have to issue the following +command:: + +>> run_project standard 4 + +.. note:: + If you want to load other targets while your project is currently running, you should use the + ``reload_all`` command (refer to :ref:`tuto-reload_cmd`) .. note:: If you want to load another data model at any time while your @@ -216,11 +233,12 @@ Send Malformed ZIP Files to the Target (Manually) How to Send a ZIP File ++++++++++++++++++++++ -In order to send a ZIP file to the target, type the following:: +In order to send a ZIP file to a loaded target, type the following:: - >> send ZIP + >> send ZIP [target ID] -which will invoke the ``unzip`` program with a ZIP file: +In our case we previously only loaded the target ID 3 (linked to the ``unzip`` program). It means that +issuing the following command with 3 as will invoke the ``unzip`` program with a ZIP file: .. code-block:: none @@ -237,6 +255,16 @@ which will invoke the ``unzip`` program with a ZIP file: ... >> +.. note:: + If you don't provide a target ID on the command line, the one that will be used will be the first + loaded one. Thus in our case, we can forget to specify the target ID. + +.. note:: + You can also send data to multiple targets at once (assuming that you enabled them at first), by + providing the list of target IDs like the following command:: + + >> send ZIP 3 5 + Note that a :class:`framework.data_model.DataModel` can define any number of data types---to model for instance the various atoms within a data format, or to represent some specific use cases, ... @@ -304,7 +332,7 @@ To send in a loop, five ZIP archives generated from the data model in a deterministic way---that is by walking through the data model---you can use the following command:: - >> send_loop 5 ZIP tWALK + >> send_loop 5 ZIP(determinist=True) tWALK We use for this example, the generic stateful disruptor ``tWALK`` whose purpose is to simply walk through the data model. Note that disruptors are @@ -316,7 +344,7 @@ Note that if you want to send data indefinitely until the generator exhausts (in or a stateful disruptor (in our case ``tWALK``) of the chain exhausts you should use ``-1`` as the number of iteration. In our case it means issuing the following command:: - >> send_loop -1 ZIP tWALK + >> send_loop -1 ZIP(determinist=True) tWALK And if you want to stop the execution before the normal termination (which could never happen if the ``finite`` parameter has not been set), then you have to issue a ``SIGINT`` signal to ``fuddly`` via @@ -401,7 +429,7 @@ Let's illustrate this with the following example: :linenos: :emphasize-lines: 1,16,19,25,30 - >> send ZIP_00 C(nb=2:path="ZIP_00/file_list/.*/file_name") tTYPE(order=True) SIZE(sz=256) + >> send ZIP_00 C(nb=2:path="ZIP_00/file_list/.*/file_name") tTYPE(max_steps=50:order=True) SIZE(sz=256) __ setup generator 'g_zip_00' __ __ setup disruptor 'd_corrupt_node_bits' __ @@ -490,22 +518,13 @@ can see on lines 16 & 19. -.. note:: Generic parameters are given to data makers - (generators/disruptors) through a tuple wrapped with the characters - ``<`` and ``>`` and separated with the character ``:``. Syntax:: - - data_maker_type - - Specific parameters are given to data makers +.. note:: + Parameters are given to data makers (generators/disruptors) through a tuple wrapped with the characters ``(`` and ``)`` and separated with the character ``:``. Syntax:: data_maker_type(param1=val1:param2=val2) - Generic and specific parameters can be used together. Syntax:: - - data_maker_type(param2=val2:param3=val3) - After ``C`` has performed its corruption, fuddly gets the result and provides it to ``tTYPE``. This disruptor is stateful, so it could @@ -566,7 +585,7 @@ exceeds 256---as the parameter ``sz`` is equal to 256. :linenos: :emphasize-lines: 1,5-7,11,16,17-18 - >> send ZIP_00 C(nb=2:path="$ZIP/file_list.*") tTYPE(order=True) SIZE(sz=256) + >> send ZIP_00 C(nb=2:path="$ZIP/file_list.*") tTYPE(max_steps=50:order=True) SIZE(sz=256) ========[ 2 ]==[ 20/08/2015 - 15:20:08 ]======================= ### Target ack received at: None @@ -688,13 +707,14 @@ Last, to avoid re-issuing the same command for each time you want to send a new data, you can use the ``send_loop`` command as follows:: - >> send_loop ZIP_00 C(nb=2:path="ZIP_00/file_list/.*") tTYPE(order=True) SIZE(sz=256) + >> send_loop ZIP_00 C(nb=2:path="ZIP_00/file_list/.*") tTYPE(max_steps=50:order=True) SIZE(sz=256) where ```` shall be replaced by the maximum number of iteration you want before fuddly return to the prompt. Note that it is a maximum; in our case it will stop at the 50 :sup:`th` run because of -``tTYPE``. - +``tTYPE``. Note that you can also use the special value -1 to loop indefinitely +or until a data maker is exhausted. +In such situation, if you want to interrupt the looping, just use ``Ctrl+C``. .. _tuto:reset-dmaker: @@ -704,7 +724,7 @@ Resetting & Cloning Disruptors Whether you want to use generators or disruptors that you previously used in a *data maker chain*, you would certainly need to reset it or to clone it. Indeed, every data maker has an internal sequencing state, -that remember if it has been disabled (and for generators it also +that remember if it has been disabled (and for generators it may also keeps the *seeds*). Thus, if you want to reuse it, one way is to reset it by issuing the following command:: @@ -717,15 +737,14 @@ You can also reset all the data makers at once by issuing the following command: >> reset_all_dmakers -.. note:: You can also choose to cleanup a *Generator* without resetting the - specifics of the previously produced data, that is, preserving the *seed* that - guided the data generation. Actually this seed is a copy of the - data that has been generated at the beginning, before any disruptor got a chance to - modify it. This original data is kept within the generator and will be provided again - if you use the command ``cleanup_dmaker`` instead of ``reset_dmaker``. The latter will - remove this seed. +.. note:: + In the case where the original data (i.e., the pristine generated data that does not get changed + by any disruptor) is asked to be preserved (for instance by using the command ``send_loop_keepseed``), + for repeatability purpose (when issuing the same command again), using the previous command will + also remove this original data. Thus you could prefer to use the command ``cleanup_dmaker`` that + will only reset the sequencing state, without resetting the seed (i.e., the original data). - Keeping such *seeds* may consume a lot of memory at some point. Moreover, they may only + Note that keeping such *seeds* may consume a lot of memory at some point. Moreover, they may only be useful for non-determinist data model. @@ -743,6 +762,8 @@ the cloned data makers:: >> send ZIP_00#new tTYPE#new +.. _tuto-reload_cmd: + Reloading Data Models / Targets / ... +++++++++++++++++++++++++++++++++++++ @@ -756,8 +777,8 @@ probes, ..., you have to reload every fuddly subsystems. To do so, you only need to issue the command ``reload_all``. Now, imagine that you want to switch to a new target already -registered, simply issue the command ``reload_all ``, where -```` is picked up through the IDs displayed by the command +registered, simply issue the command ``reload_all [target_ID1 .. target_ID2]``, where +``target IDs`` are picked up through the IDs displayed by the command ``show_targets`` Finally, if you want to switch to a new data model while a project is @@ -815,7 +836,7 @@ This command will display the `following <#operator-show>`_: To launch the operator ``Op1`` and limit to 5 the number of test cases to run, issue the command:: - >> launch_operator Op1 + >> launch_operator Op1(max_steps=5) This will trigger the Operator that will execute the ``display`` program with the first generated JPG file. It will look at ``stdout`` @@ -826,7 +847,7 @@ also try to avoid saving JPG files that trigger errors whose type has already been seen. Once the operator is all done with this first test case, it can plan the next actions it needs ``fuddly`` to perform for it. In our case, it will go on with the next iteration of a disruptor -chain, basically ``JPG tTYPE``. +chain, basically ``JPG(finite=True) tTYPE``. Replay Data From a Previous Session @@ -879,15 +900,12 @@ will need to issue the following commands: .. code-block:: python :linenos: - :emphasize-lines: 1,2,5 from framework.plumbing import * fmk = FmkPlumbing() - -The lines 1, 2 and 5 are not necessary if you don't intend to use -external libraries. From now on you can use ``fuddly`` through the +From now on you can use ``fuddly`` through the object ``fmk``. Every commands defined by ``Fuddly Shell`` (refer to :ref:`tuto:start-fuzzshell`) are backed by a method of the class :class:`framework.plumbing.FmkPlumbing`. @@ -911,7 +929,7 @@ Here under some basic commands to start with: fmk.show_targets() # Select the target with ID ``3`` - fmk.set_target(3) + fmk.load_targets(3) # To show all the available data models fmk.show_data_models() @@ -967,7 +985,7 @@ Here under some basic commands to start with: # Perform a tTYPE disruption on it, but give the 5th generated # cases and enforce the disruptor to strictly follow the ZIP structure # Finally truncate the output to 200 bytes - action_list = [('tTYPE', UI(init=5), UI(order=True)), ('SIZE', None, UI(sz=200))] + action_list = [('tTYPE', UI(init=5, order=True)), ('SIZE', UI(sz=200))] altered_data = fmk.get_data(action_list, data_orig=Data(dt)) # Send this new data and look at the actions that perform tTYPE and @@ -1935,6 +1953,7 @@ another inappropriate separator. new_values.remove(orig_val) node.import_value_type(value_type=vtype.String(values=new_values)) + node.unfreeze() node.make_finite() node.make_determinist() @@ -2038,7 +2057,7 @@ show the beginning of ``generic/standard_proj.py``: # If you only want one default DM, provide its name directly as follows: # project.default_dm = 'mydf' - logger = Logger(export_data=False, explicit_data_recording=False, + logger = Logger(record_data=False, explicit_data_recording=False, export_orig=False, export_raw_data=False) printer1_tg = PrinterTarget(tmpfile_ext='.png') @@ -2067,12 +2086,18 @@ show the beginning of ``generic/standard_proj.py``: targets = [local_tg, local2_tg, local3_tg, printer1_tg, net_tg] -A project file should contain at a minimum a -:class:`framework.project.Project` object (referenced by a variable -``project``), a :class:`framework.logger.Logger` object -(:ref:`logger-def`, referenced by a variable ``logger``), and -optionally target objects (referenced by a variable ``targets``, -:ref:`targets-def`), and operators & probes (:ref:`tuto:operator`). +A project file should contain at a minimum: + +- a :class:`framework.project.Project` object (referenced by a variable ``project``) +- a :class:`framework.logger.Logger` object (:ref:`logger-def`, referenced by a variable ``logger``) + +and optionally: + +- targets (referenced by a variable ``targets``, :ref:`targets-def`) +- scenarios (:ref:`scenario-infra`) that can be registered into a project through the method + :meth:`framework.project.Project.register_scenarios` +- probes (:ref:`tuto:probes`). +- operators (:ref:`tuto:operator`). A default data model or a list of data models can be added to the project through its attribute ``default_dm``. ``fuddly`` will use this @@ -2176,7 +2201,7 @@ The outputs of the logger are of four types: Some parameters allows to customize the behavior of the logger, such as: -- ``export_data`` which control the location of where data +- ``record_data`` which control the location of where data will be stored. If set to ``False``, instead of being stored in separate files as explained previously, they will be written directly within the log files (if ``enable_file_logging`` is set to ``True``). @@ -2299,7 +2324,7 @@ is given here under: if fmk_feedback.is_flag_set(FmkFeedback.NeedChange): op.set_flag(Operation.Stop) else: - actions = [('SEPARATOR', UI(determinist=True)), ('tSTRUCT', None, UI(deep=True))] + actions = [('SEPARATOR', UI(determinist=True)), ('tSTRUCT', UI(deep=True))] op.add_instruction(actions) return op @@ -2421,7 +2446,7 @@ information from the target is given here under: status_code = check(target) - self.pstatus.set_status(status_code) + self.pstatus.value = status_code if status_code < 0: self.status.set_private_info("Something is wrong with the target!") @@ -2515,7 +2540,7 @@ Let's illustrate this with the following example: health_status = monitor.get_probe_status(health_check) - if health_status.get_status() < 0 and self.op_state == 'critical': + if health_status.value < 0 and self.op_state == 'critical': linst.set_instruction(LastInstruction.RecordData) linst.set_operator_feedback('Data sent seems worthwhile!') linst.set_operator_status(-3) diff --git a/framework/basic_primitives.py b/framework/basic_primitives.py index 7a6e824..4cf147d 100644 --- a/framework/basic_primitives.py +++ b/framework/basic_primitives.py @@ -33,6 +33,11 @@ def rand_string(size=None, min=1, max=10, str_set=string.printable): out = "" if size is None: size = random.randint(min, max) + else: + # if size is not an int, TypeError is raised with python3, but not + # with python2 where the loop condition is always evaluated to True + assert isinstance(size, int) + while len(out) < size: val = random.choice(str_set) out += val diff --git a/framework/config.py b/framework/config.py new file mode 100644 index 0000000..8e2ffe7 --- /dev/null +++ b/framework/config.py @@ -0,0 +1,597 @@ +############################################################################## +# +# Copyright 2017 Matthieu Daumas +# +############################################################################## +# +# This file is part of fuddly. +# +# fuddly is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# fuddly is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with fuddly. If not, see +# +############################################################################## + +# TODO: dump the whole code +# TODO: rewrite a proper module without configparser, nor "textual" backend +# TODO: write a backend to write config objects to files +# TODO: document it +# +# Features: +# - configparser-enabled (sic) +# - transparent access to the config keys +# - inline documentation of the config keys +# - recursive pattern/mergeable configs +# + +import io +import os +import re +import sys + +try: + import configparser +except BaseException: + import ConfigParser as configparser + +reserved = {'config_name', 'parser', 'help', 'write', 'global'} +verbose = False + + +class default: + def __init__(self): + self.configs = {} + + __unindent = re.compile(r'^;;\s\s*', re.MULTILINE) + + def _unindent(self, multiline): + return self.__unindent.sub('', multiline) + + def add(self, name, doc): + if sys.version_info[0] <= 2: + doc = unicode(doc) + + self.configs[name] = self._unindent(doc) + + +default = default() + +default.add('FmkPlumbing', u''' +[global] +config_name: FmkPlumbing + +[defvalues] +fuzz.delay = 0.01 +fuzz.burst = 1 + +;; [defvalues.doc] +;; self: (default values used when the framework resets) +;; fuzz.delay: Default value (> 0) for fuzz_delay +;; fuzz.burst: Default value (>= 1)for fuzz_burst + +''') + +default.add('FmkShell', u''' +[global] +config_name: FmkShell +prompt: >> + +;; [global.doc] +;; prompt: Set the 'Fuddly Shell' prompt + +[config] +middle: 40 +indent.width: 4 +indent.level: 0 + +;; [config.doc] +;; self: Configuration applicable to the 'config' command +;; middle: Set the column where the helpers are defined. +;; indent.width: Set the indentation width used to display the helpers. +;; indent.level: Set the initial level of indentation width + used to display the helpers. + +[send_loop] +aligned: True +aligned_options.batch_mode: False +aligned_options.hide_cursor: True +aligned_options.prompt_height: 3 + +;; [send_loop.doc] +;; self: Configuration applicable to the 'send_loop' command. +;; +;; aligned: Enable aligned display while sending data payloads. +;; aligned_options.batch_mode: Enable fitting multiple payloads onscreen + (when using 'send_loop -1 '). +;; aligned_options.hide_cursor: Attempt to reduce blinking by hiding cursor. +;; aligned_options.prompt_height: Estimation of prompt's height. + +''') + + +def check_type(name, attr, value): + original = value + + try: + attr = str(attr) + except Exception as e: + raise AttributeError("unable to cast key's value " + + "'{}' into a string".format(name) + ': ' + str(e)) + + try: + value = str(value) + except Exception as e: + raise AttributeError(( + "unable to cast '{}' " + "for key '{}' into a string").format( + value, name) + ': ' + str(e)) + + test, value = try_bool(attr, value, name) + if test is not None: + return value + + test, value = try_int(attr, value, name) + if test is not None: + return value + + test, value = try_float(attr, value, name) + if test is not None: + return value + + return original + + +def try_bool(attr, value, name): + booleans = [u'True', u'False'] + if attr in booleans: + test = (attr == u'True') + else: + test = None + if test is not None: + if value in booleans: + return (test, (value == u'True')) + else: + raise AttributeError("key '{}' expects a boolean".format(name)) + + return (test, value) + + +def try_int(attr, value, name): + try: + test = int(attr) + except BaseException: + test = None + if test is not None: + try: + return (test, int(value)) + except BaseException: + raise AttributeError("key '{}' expects an integer".format(name)) + + return (test, value) + + +def try_float(attr, value, name): + try: + test = float(attr) + except BaseException: + test = None + if test is not None: + try: + return (test, float(value)) + except BaseException: + raise AttributeError("key '{}' expects a float".format(name)) + + return (test, value) + + +def config_write(that, stream=sys.stdout): + + that_dict = object.__getattribute__(that, '__dict__') + subconfigs = [] + for item in that_dict.items(): + if isinstance(item[1], config): + subconfigs.append(item) + + for name, subconfig in subconfigs: + setattr(that, name, subconfig) + + return that.parser.write(stream) + + +def sectionize(that, parent): + if sys.version_info[0] > 2: + unicode = str + if parent is not None: + that.config_name = parent + + try: + name = unicode(that.config_name) + except BaseException: + name = that.config_name + + parser = configparser.ConfigParser() + resection = re.compile(r'^([^.]*)\.?(.*)') + for section in that.parser.sections(): + match = resection.match(section) + if not match: + raise RuntimeError("unable to match '{}'".format(section) + + ' section name') + + if len(match.group(2)) > 0: + target = name + '.' + match.group(2) + else: + target = name + + if not parser.has_section(target): + parser.add_section(target) + + if match.group(1) == 'global': + for option in that.parser.options(section): + value = that.parser.get(section, option) + parser.set(target, option, value) + else: + for option in that.parser.options(section): + value = that.parser.get(section, option) + parser.set(target, match.group(1) + '.' + option, value) + + return parser + + +def get_help_format(line, doc, level, indent, middle): + if doc is None or len(doc) < 1: + doc = '\n' + + try: + line = str(line) + except BaseException: + pass + + msg = '' + space = ' ' + lines = line.splitlines(True) + for line in lines: + line = level * indent * space + line + msg += line + + fst = True + docs = doc.splitlines(True) + for doc in docs: + if fst: + size = middle - len(line) + msg += size * space + doc + fst = False + else: + msg += middle * space + doc + + if not ('\n' in doc): + msg += '\n' + return msg + + +def get_help_attr(that, name, level=0, indent=4, middle=40): + if name in reserved: + return get_help_format(name + ': ', + '(implementation details, reserved)', level, + indent, middle) + + dot_section = re.compile(r'^[^.]+\.[^.]+$') + if dot_section.match(name) and that.parser.has_section(name): + return get_help_format(name + ': ', + '(special section, reserved)', level, indent, + middle) + + if that.parser.has_section(name): + try: + doc = that.parser.get(name + '.doc', 'that') + except BaseException: + doc = None + + msg = '\n' + msg += get_help_format(name + ':', doc, level, indent, middle) + return (msg + get_help( + getattr(that, name), None, level + 1, indent, middle)) + + try: + doc = that.parser.get('global.doc', name) + except BaseException: + doc = None + + try: + try: + value = str(that.parser.get('global', name)) + except BaseException: + value = that.parser.get('global', name) + except BaseException: + kdict = object.__getattribute__(that, '__dict__') + keys = [key for key in kdict] + if name in keys and isinstance(kdict[name], config_dot_proxy): + msg = name + ': (subkeys)\n' + keys = [key for key in keys if key.startswith(name + '.')] + for key in sorted(keys): + msg += get_help_attr(that, key, level + 1, indent, middle) + return msg + else: + value = '' + + return get_help_format(name + ': ' + value, doc, level, indent, middle) + + +def get_help(that, name=None, level=0, indent=4, middle=40): + if name is not None: + return get_help_attr(that, name, level, indent, middle) + + msg = '' + resection = re.compile(r'^[^.]*$') + for section in that.parser.sections(): + if section == 'global': + for option in that.parser.options(section): + if option in reserved: + continue + else: + msg += get_help_attr(that, option, level, indent, middle) + elif resection.match(section): + msg += get_help_attr(that, section, level, indent, middle) + else: + pass + + return msg + + +def config_setattr(that, name, value): + try: + attr = object.__getattribute__(that, name) + except BaseException: + attr = None + + if not isinstance(value, config): + if isinstance(value, config_dot_proxy): + raise RuntimeError("'{}' (config_dot_proxy)".format(name) + + ' can not be used as value.') + + if attr is not None: + value = check_type(name, attr, value) + + if '.' in name: + prefixes = name.split('.') + for i in range(1, len(prefixes)): + prefix = '.'.join(prefixes[:-1 * i]) + proxy = config_dot_proxy(that, prefix) + object.__setattr__(that, prefix, proxy) + + strvalue = str(value) + that.parser.set('global', name, strvalue) + return object.__setattr__(that, name, value) + + if attr is None: + flat_parser = sectionize(value, name) + for section in flat_parser.sections(): + if not that.parser.has_section(section): + that.parser.add_section(section) + + for option in flat_parser.options(section): + if option in reserved: + continue + subvalue = flat_parser.get(section, option) + that.parser.set(section, option, subvalue) + + return object.__setattr__(that, name, value) + + if not isinstance(attr, config): + raise AttributeError("unable to replace key '{}'".format(name) + + ' by a section') + + attr_parser = sectionize(attr, name) + for section in attr_parser.sections(): + if not that.parser.has_section(section): + continue + + for option in attr_parser.options(section): + if option in reserved: + continue + + that.parser.remove_option(section, option) + + if len(that.parser.options(section)) < 1: + that.parser.remove_section(section) + + object.__setattr__(that, name, None) + return setattr(that, name, value) + + +def config_getattribute(that, name): + if name == 'help': + + def get_help_proxy(name=None, level=0, indent=4, middle=40): + msg = get_help(that, name, level, indent, middle) + try: + return str(msg) + except BaseException: + return msg + + return get_help_proxy + + if name == 'write': + + def write_proxy(stream=sys.stdout): + return config_write(that, stream) + + return write_proxy + + try: + attr = object.__getattribute__(that, name) + try: + attr = check_type(name, attr, attr) + except BaseException: + pass + except BaseException: + attr = None + + if attr is None or '__' in name[:2] or '_config__' in name[:9]: + msg = "'config' instance for '{}' has no key nor section '{}'" + if not name == 'config_name': + raise AttributeError(msg.format(that.config_name, name)) + else: + raise AttributeError(msg.format('error', name)) + + return attr + + +class config_dot_proxy(object): + def __init__(self, config, prefix): + object.__setattr__(self, 'parent', config) + object.__setattr__(self, 'prefix', prefix) + + def __getattribute__(self, name): + parent = object.__getattribute__(self, 'parent') + prefix = object.__getattribute__(self, 'prefix') + try: + return getattr(parent, prefix + '.' + name) + except BaseException: + return config_dot_proxy(self, name) + + def __setattr__(self, name, value): + parent = object.__getattribute__(self, 'parent') + prefix = object.__getattribute__(self, 'prefix') + return setattr(parent, prefix + '.' + name, value) + + +class config(object): + def __init__(self, parent, path=['.'], ext=['.ini', '.conf', '.cfg']): + + object.__setattr__(self, 'parser', configparser.ConfigParser()) + + if isinstance(parent, str): + name = parent + else: + try: + name = parent.__class__.__name__ + except BaseException: + name = parent.__name__ # (no parent.__class__.__name__) + + loaded = False + for pdir in path: + if not os.path.isdir(pdir): + continue + for pext in ext: + filename = os.path.join(pdir, name + pext) + if not os.path.isfile(filename): + continue + + try: + if verbose: + sys.stderr.write('Loading {}...\n'.format(filename)) + with open(filename, 'r') as cfile: + if sys.version_info[0] > 2: + self.parser.read_file(cfile, source=filename) + else: + self.parser.readfp(cfile, filename=filename) + loaded = True + except BaseException as e: + if verbose: + sys.stderr.write('Warning: Unable to load {}: '.format( + filename) + str(e) + '\n') + continue + + if loaded or name in default.configs: + if not loaded and sys.version_info[0] > 2: + if verbose: + sys.stderr.write( + 'Loading default config for {}...\n'.format(name)) + self.parser.read_string(default.configs[name], + 'default_' + name) + loaded = True + elif not loaded: + if verbose: + sys.stderr.write( + 'Loading default config for {}...\n'.format(name)) + stream = io.StringIO() + stream.write(default.configs[name]) + stream.seek(0) + self.parser.readfp(stream, 'default_' + name) + loaded = True + + if not self.parser.has_section('global'): + raise AttributeError( + ("default config '{}' has no '{}' section").format( + name, 'global')) + + try: + self.parser.get('global', 'config_name') + except BaseException: + raise AttributeError( + ("default config '{}' has no '{}' key").format( + name, 'config_name')) + + resection = re.compile(r'^([^.]*)(\..*)?') + for section in self.parser.sections(): + + match = resection.match(section) + if not match: + raise RuntimeError( + ("unable to match '{}' section name").format(section)) + + if not section == 'global': + if match.group(2) and len(match.group(2)) == 0: + setattr(self, section, config(section)) + else: + try: + subconfig = object.__getattribute__( + self, match.group(1)) + except BaseException: + setattr(self, + match.group(1), config(match.group(1))) + subconfig = object.__getattribute__( + self, match.group(1)) + + try: + subconfig.parser.add_section( + 'global' + match.group(2)) + except BaseException: + pass + + for option in self.parser.options(section): + value = self.parser.get(section, option) + if section == 'global': + if self.parser.has_section(option): + raise AttributeError( + ("default config '{}' " + + "has global '{}' which " + + "overrides '{}' section").format( + name, option, option)) + setattr(self, option, value) + else: + if match.group(2) and len(match.group(2)) > 0: + subconfig = object.__getattribute__( + self, match.group(1)) + subconfig.parser.set('global' + match.group(2), + option, value) + else: + subconfig = object.__getattribute__(self, section) + setattr(subconfig, option, value) + + if not loaded and verbose: + sys.stderr.write("Creating a new config for {}...\n".format(name)) + + if not self.parser.has_section('global'): + self.parser.add_section('global') + + try: + object.__getattribute__(self, 'config_name') + except BaseException: + self.config_name = 'global' + + def __getattribute__(self, name): + config_get = config_getattribute + return config_get(self, name) + + def __setattr__(self, name, value): + config_set = config_setattr + return config_set(self, name, value) diff --git a/framework/cosmetics.py b/framework/cosmetics.py new file mode 100644 index 0000000..520943f --- /dev/null +++ b/framework/cosmetics.py @@ -0,0 +1,392 @@ +############################################################################## +# +# Copyright 2017 Matthieu Daumas +# +############################################################################## +# +# This file is part of fuddly. +# +# fuddly is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# fuddly is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with fuddly. If not, see +# +############################################################################## +"""Handle fuddly shell's cosmetics, like curses-related functionnalities.""" + +import contextlib +import io +import os +import re +import sys + +#: bool: internals, check for an ImportError for curses & termios +import_error = False +try: + import curses +except ImportError as e: + sys.stderr.write('WARNING [FMK]: python(3)-curses unavailable, ' + + 'raw fuddly shell only.') + sys.stderr.write(str(e)) + import_error = True + +try: + import termios +except ImportError as e: + sys.stderr.write('WARNING [FMK]: POSIX termios unavailable, ' + + 'restricted terminal capabilities.') + sys.stderr.write(str(e)) + import_error = True + + +def setup_term(): + # type: () -> None + """Handle curses vs readline issues + + See `issue 2675`__ for further informations, enables resizing + the terminal. + + __ https://bugs.python.org/issue2675""" + + if import_error: + return + + # unset env's LINES and COLUMNS to trigger a size update + os.unsetenv('LINES') + os.unsetenv('COLUMNS') + + # curses's setupterm with the real output (sys.__stdout__) + try: + curses.setupterm(fd=sys.__stdout__.fileno()) + except curses.error as e: + print('\n*** WARNING: {!s}'.format(e)) + print(' --> set $TERM variable manually to xterm-256color') + os.environ['TERM'] = 'xterm-256color' + curses.setupterm(fd=sys.__stdout__.fileno()) + + +#: file: A reference to the "unwrapped" stdout. +stdout_unwrapped = sys.stdout + +#: file or io.BytesIO or io.TextIOWrapper: the buffer that will replace stdout +stdout_wrapped = None +if import_error: + stdout_wrapped = sys.__stdout__ + + +class wrapper: + """Wrapper class, handle cosmetics internals while wrapping stdout.""" + + def __init__(self, parent_wrapper): + self.countp = 0 + self.initial = True + self.page_head = re.compile(r'\n') + self.batch_mode = False + self.prompt_height = 0 + + self.parent_write = parent_wrapper.write + + def write(self, payload, *kargs, **kwargs): + if self.page_head.match(payload): + try_flush(batch=self.batch_mode) + + return self.parent_write(self, payload, *kargs, **kwargs) + + def reinit(self): + """Reset the wrapper to its initial state while keeping configuration. + + (only resets the page counter)""" + + self.countp = 0 + + +if sys.version_info[0] > 2: + + class stdout_wrapper(wrapper, io.TextIOWrapper): + """Wrap stdout and handle cosmetic issues.""" + + def __init__(self): + io.TextIOWrapper.__init__(self, + io.BytesIO(), sys.__stdout__.encoding) + wrapper.__init__(self, io.TextIOWrapper) + + def reinit(self): + wrapper.reinit(self) + io.TextIOWrapper.__init__(self, io.BytesIO()) + + stdout_wrapped = stdout_wrapper() +else: + + class stdout_wrapper(wrapper, io.BytesIO): + """Wrap stdout and handle cosmetic issues.""" + + def __init__(self): + io.BytesIO.__init__(self) + wrapper.__init__(self, io.BytesIO) + + def reinit(self): + wrapper.reinit(self) + io.BytesIO.__init__(self) + + stdout_wrapped = stdout_wrapper() + +if not import_error: + + # (call curses's setupterm at least one time) + setup_term() + + el = curses.tigetstr("el") + ed = curses.tigetstr("ed") + cup = curses.tparm(curses.tigetstr("cup"), 0, 0) + civis = curses.tigetstr("civis") + cvvis = curses.tigetstr("cvvis") +else: + if sys.version_info[0] > 2: + el, ed, cup, civis, cvvis = ('', ) * 5 + else: + el, ed, cup, civis, cvvis = (b'', ) * 5 + + +def get_size( + cutby=(0, 0), # type: Tuple[int, int] + refresh=True # type: bool +): + # type: (...) -> Tuple[int, int] + """Returns the terminal size as a (width, height) tuple + + Args: + refresh (bool): Try to refresh the terminal's size, + required if you use readline. + cutby (Tuple[int, int]): + Cut the terminal size by an offset, the first integer + of the tuple correspond to the width, the second to the + height of the terminal. + + Returns: + The (width, height) tuple corresponding to the terminal's + size (reduced slightly by the :literal:`cutby` argument). + The minimal value for the width or the height is 1. + If curses is not available, returns (79, 63).""" + + if import_error: + return (79, 63) + + # handle readline/curses interactions + if refresh: + setup_term() + + # retrieve the terminal's size: + # - if refresh, initiate a curses window for an updated size, + # - else, retrieve it via a numeric capability. + # + if refresh: + height, width = curses.initscr().getmaxyx() + curses.endwin() + else: + height = curses.tigetnum("lines") + width = curses.tigetnum("cols") + + # now *cut* the terminal by the specified offset + width -= cutby[0] + height -= cutby[1] + + # handle negative values + if width < 2: + width = 1 + if height < 2: + height = 1 + + # return the tuple + return (width, height) + + +def buffer_content(): + # type: () -> bytes + """Returns stdout_wrapped's content + + Returns: + The whole content of stdout_wrapped.""" + + if sys.version_info[0] > 2: + stdout_wrapped.seek(0) + payload = stdout_wrapped.buffer.read() + else: + payload = stdout_wrapped.getvalue() + return payload + + +def estimate_nblines(width # type: int + ): + # type: (...) -> int + """Estimate the number of lines in the buffer + + Args: + width (int): width of the terminal, used to calculate lines + wrapping in the wrapped stdout. + + Returns: + The estimated number of lines that the payload will take on + screen.""" + + nblines = 0 + payload = buffer_content() + + lines = payload.splitlines() + for line in lines: + length = len(line) + nblines += length // width + 1 + return nblines + 1 + + +def disp( + payload, # type: bytes +): + # type: (...) -> None + """Display on stdout the bytes passed. + + Args: + payload (bytes): the bytes displayed on screen.""" + + if sys.version_info[0] > 2: + stdout_unwrapped.buffer.write(payload) + else: + stdout_unwrapped.write(payload) + + +def tty_noecho(): + # type: () -> None + """Disable echo mode in the tty.""" + + # (we use POSIX tty, as we do not use curses for everything) + fd = sys.__stdout__.fileno() + try: + flags = termios.tcgetattr(fd) + flags[3] = flags[3] & ~termios.ECHO + termios.tcsetattr(fd, termios.TCSADRAIN, flags) + except BaseException: + pass + + # (we hide the cursor for a nicer display) + disp(civis) + + +def tty_echo(): + # type: () -> None + """Re-enable echo mode for the tty.""" + + # (we use POSIX tty, as we do not use curses for everything) + fd = sys.__stdout__.fileno() + try: + flags = termios.tcgetattr(fd) + flags[3] = flags[3] | termios.ECHO + termios.tcsetattr(fd, termios.TCSADRAIN, flags) + except BaseException: + pass + + # (we unhide the cursor and clean the line for nicer display) + disp(cvvis + ed) + + +def restore_stdout(): + # type: () -> None + """Restore sys.stdout and the terminal.""" + + sys.stdout = stdout_unwrapped + try_flush(force=True) + tty_echo() + + +def try_flush( + batch=False, # type: bool + force=False # type: bool +): + # type: (...) -> None + """Display buffered lines on screen taking into account various factors. + + Args: + batch (bool): try to put as much as possilbe text on screen. + force (bool): force the buffer to output its content.""" + + # retrieve the terminal size + width, height = get_size(cutby=(0, stdout_wrapped.prompt_height)) + + # flush the buffer, then estimate the number of lines + stdout_wrapped.flush() + nblines = estimate_nblines(width) + + # batch mode needs to estimate payloads size (skipped if force) + if (not force) or batch: + if stdout_wrapped.countp > 0: + avg_size_per_payload = nblines // stdout_wrapped.countp + else: + avg_size_per_payload = nblines + stdout_wrapped.countp += 1 + + # (if force, or non-batch, or sufficient output, display it) + if (force or (not batch) or nblines + avg_size_per_payload > height): + + # protect history by padding with line feeds + if stdout_wrapped.initial: + stdout_wrapped.initial = False + disp(b'\n' * (height + nblines + 1)) + + # use `el` term capabilitie to wipe endlines as we display + payload = buffer_content() + payload = payload.replace(b'\n', el + b'\n') + + # if not force (continuous display), we erase the first + # payload (to have a log entry without disturbing scrolling + # nor getting a blinking terminal), then we display the + # payload a second time (in order to see it on screen), else + # (force == True), then we have the last payload to display, + # no need to duplicate it with unnecessary buffering. + # + if not force: + pad = b'\n' * (height - nblines + stdout_wrapped.prompt_height) + padded_payload = cup + payload * 2 + pad + else: + padded_payload = cup + payload + + disp(padded_payload) + + # empty the buffer, reset the payload counter + stdout_wrapped.reinit() + + # if it is the last payload, reenable echo-ing. + if force: + tty_echo() + + +@contextlib.contextmanager +def aligned_stdout( + enabled, # type: bool + page_head, # type: str + batch_mode, # type: bool + hide_cursor, # type: bool + prompt_height # type: int +): + # type: (...) -> None + """do_send_loop cosmetics, contextualize stdout's wrapper.""" + + if enabled: + if hide_cursor: + tty_noecho() + + sys.stdout = stdout_wrapped + stdout_wrapped.prompt_height = prompt_height + stdout_wrapped.batch_mode = batch_mode + stdout_wrapped.page_head = re.compile(page_head) + stdout_wrapped.initial = True + + yield + restore_stdout() + else: + yield diff --git a/framework/data.py b/framework/data.py index 780904f..39ae93e 100644 --- a/framework/data.py +++ b/framework/data.py @@ -201,7 +201,8 @@ class Data(object): _empty_data_backend = EmptyBackend() - def __init__(self, data=None): + def __init__(self, data=None, altered=False, tg_ids=None): + self.altered = altered self._backend = None self._type = None @@ -216,6 +217,8 @@ def __init__(self, data=None): self.info_list = [] self.info = {} + self.scenario_dependence = None + # callback related self._callbacks = {} self._pending_ops = {} @@ -226,6 +229,8 @@ def __init__(self, data=None): # If it comes from a scenario _origin point to the related scenario. self._origin = None + self.tg_ids = tg_ids # targets ID + # This attribute is set to True when the Data content has been retrieved from the fmkDB self.from_fmkdb = False @@ -240,6 +245,22 @@ def __init__(self, data=None): def content(self): return self._backend.content + @property + def tg_ids(self): + return self._targets + + @tg_ids.setter + def tg_ids(self, tg_ids): + if tg_ids is None: + self._targets = None + elif isinstance(tg_ids, list): + assert len(tg_ids) > 0 + self._targets = tg_ids + elif isinstance(tg_ids, int): + self._targets = [tg_ids] + else: + raise ValueError + def is_empty(self): return isinstance(self._backend, EmptyBackend) @@ -301,6 +322,7 @@ def generate_info_from_content(self, original_data=None, origin=None, additional dmaker_name = self._backend.data_maker_name if original_data is not None: + self.altered = original_data.altered if original_data.origin is not None: self.add_info("Data instantiated from: {!s}".format(original_data.origin)) if original_data.info: @@ -468,6 +490,7 @@ def __copy__(self): new_data._callbacks[hook][id(cbk)] = cbk new_data._pending_ops = {} # we do not copy pending_ops new_data._backend = copy.copy(self._backend) + new_data._targets = copy.copy(self._targets) return new_data def __str__(self): diff --git a/framework/data_model.py b/framework/data_model.py index 0760a55..0e3e96e 100644 --- a/framework/data_model.py +++ b/framework/data_model.py @@ -38,6 +38,8 @@ class DataModel(object): file_extension = 'bin' name = None + knowledge_source = None + def pre_build(self): """ This method is called when a data model is loaded. @@ -56,15 +58,33 @@ def build_data_model(self): pass - def absorb(self, raw_data, idx): + def create_node_from_raw_data(self, data, idx, filename): """ If your data model is able to absorb raw data, do it here. This function is called for each files (with the right extension) - present in imported_data/. - - It should return an modeled data (atom) + present in ``imported_data/``. + + Args: + filename (str): name of the imported file + data (bytes): file content + idx (int): index of the imported file + + Returns: + :class:`framework.node.Node`: a modeled data (atom) or ``None`` + + """ + return data + + def validation_tests(self): """ - return raw_data + Optional test cases to validate the correct behavior of the data model + + Returns: + bool: ``True`` if the validation succeeds. ``False`` otherwise + + """ + + return True def cleanup(self): pass @@ -86,6 +106,7 @@ def __str__(self): def register(self, *atom_list): for a in atom_list: + if a is None: continue key, prepared_atom = self._backend(a).prepare_atom(a) self._dm_hashtable[key] = prepared_atom @@ -141,7 +162,7 @@ def import_file_contents(self, extension=None, absorber=None, subdir=None, path=None, filename=None): if absorber is None: - absorber = self.absorb + absorber = self.create_node_from_raw_data if extension is None: extension = self.file_extension @@ -170,7 +191,7 @@ def is_good_file_by_fname(fname): for name in files: with open(os.path.join(path, name), 'rb') as f: buff = f.read() - d_abs = absorber(buff, idx) + d_abs = absorber(buff, idx, name) if d_abs is not None: msgs[name] = d_abs idx +=1 @@ -195,6 +216,19 @@ class NodeBackend(object): def __init__(self, data_model): self._dm = data_model self._confs = set() + # self._knowledge_source = None + + # @property + # def knowledge_source(self): + # return self._knowledge_source + # + # @knowledge_source.setter + # def knowledge_source(self, src): + # self._knowledge_source = src + # + # def update_knowledge_source(self, atom): + # if self.knowledge_source is not None: + # atom.env.knowledge_source = self.knowledge_source def merge_with(self, node_backend): self._confs = self._confs.union(node_backend._confs) @@ -223,7 +257,7 @@ def prepare_atom(self, atom): if atom.env is None: self.update_atom(atom) else: - atom.env.set_data_model(self._dm) + self.update_atom(atom, existing_env=True) self._confs = self._confs.union(atom.gather_alt_confs()) @@ -231,13 +265,15 @@ def prepare_atom(self, atom): def atom_copy(self, orig_atom, new_name=None): name = orig_atom.name if new_name is None else new_name - return Node(name, base_node=orig_atom, ignore_frozen_state=False, new_env=True) + node = Node(name, base_node=orig_atom, ignore_frozen_state=False, new_env=True) + # self.update_knowledge_source(node) + return node - def update_atom(self, atom): - env = Env() - env.set_data_model(self._dm) - atom.set_env(env) + def update_atom(self, atom, existing_env=False): + if not existing_env: + atom.set_env(Env()) + atom.env.set_data_model(self._dm) + # self.update_knowledge_source(atom) def get_all_confs(self): return sorted(self._confs) - diff --git a/framework/database.py b/framework/database.py index e64d38c..29ff58d 100644 --- a/framework/database.py +++ b/framework/database.py @@ -30,6 +30,7 @@ import framework.global_resources as gr import libs.external_modules as em +from framework.knowledge.feedback_collector import FeedbackSource from libs.external_modules import * from libs.utils import ensure_dir, chunk_lines @@ -50,7 +51,7 @@ def regexp_bin(expr, item): return robj is not None -class FeedbackHandler(object): +class FeedbackGate(object): def __init__(self, database): """ @@ -63,13 +64,24 @@ def __iter__(self): for item in self.db.iter_last_feedback_entries(): yield item + def get_feedback_from(self, source): + if not isinstance(source, FeedbackSource): + source = FeedbackSource(source) + + try: + fbk = self.db.last_feedback[source] + except KeyError: + raise + else: + return fbk + def iter_entries(self, source=None): """ Iterate over feedback entries that are related to the last data which has been sent by the framework. Args: - source ('str'): name of the feedback source to consider + source (FeedbackSource): feedback source to consider Returns: python generator: A generator that iterates over all the requested feedback entries and provides for each: @@ -82,16 +94,16 @@ def iter_entries(self, source=None): for item in self.db.iter_last_feedback_entries(source=source): yield item - def sources(self): + def sources_names(self): """ Return a list of the feedback source names related to the last data which has been sent by the framework. Returns: - list: feedback sources + list: names of the feedback sources """ - return self.db.last_feedback.keys() + return [str(fs) for fs in self.db.last_feedback.keys()] # for python2 compatibility def __nonzero__(self): @@ -123,6 +135,8 @@ def __init__(self, fmkdb_path=None): # self._cur = None self.enabled = False + self.current_project = None + self.last_feedback = {} self._data_id = None @@ -189,10 +203,12 @@ def _sql_handler(self): connection.create_function("REGEXP", 2, regexp) connection.create_function("BINREGEXP", 2, regexp_bin) - while not self._sql_handler_stop_event.is_set(): + while True: with self._sql_stmt_submitted_cond: - self._sql_stmt_submitted_cond.wait(0.01) + if self._sql_handler_stop_event.is_set() and not self._sql_stmt_list: + break + self._sql_stmt_submitted_cond.wait(0.001) if self._sql_stmt_list: sql_stmts = self._sql_stmt_list @@ -234,7 +250,7 @@ def _sql_handler(self): self._sql_stmt_handled.set() - self._sql_handler_stop_event.wait(0.01) + self._sql_handler_stop_event.wait(0.001) if connection: connection.close() @@ -307,6 +323,7 @@ def disable(self): def flush_current_feedback(self): self.last_feedback = {} + self.last_feedback_sources_names = {} def execute_sql_statement(self, sql_stmt, params=None): return self.submit_sql_stmt(sql_stmt, params=params, outcome_type=Database.OUTCOME_DATA) @@ -337,7 +354,7 @@ def insert_dmaker(self, dm_name, dtype, name, is_gen, stateful, clone_type=None) def insert_data(self, dtype, dm_name, raw_data, sz, sent_date, ack_date, - target_name, prj_name, group_id=None): + target_ref, prj_name, group_id=None): if not self.enabled: return None @@ -347,13 +364,13 @@ def insert_data(self, dtype, dm_name, raw_data, sz, sent_date, ack_date, stmt = "INSERT INTO DATA(GROUP_ID,TYPE,DM_NAME,CONTENT,SIZE,SENT_DATE,ACK_DATE,"\ "TARGET,PRJ_NAME)"\ " VALUES(?,?,?,?,?,?,?,?,?)" - params = (group_id, dtype, dm_name, blob, sz, sent_date, ack_date, target_name, prj_name) + params = (group_id, dtype, dm_name, blob, sz, sent_date, ack_date, str(target_ref), prj_name) err_msg = 'while inserting a value into table DATA!' if self._data_id is None: - did = self.submit_sql_stmt(stmt, params=params, outcome_type=Database.OUTCOME_ROWID, + d_id = self.submit_sql_stmt(stmt, params=params, outcome_type=Database.OUTCOME_ROWID, error_msg=err_msg) - self._data_id = did + self._data_id = d_id else: self.submit_sql_stmt(stmt, params=params, error_msg=err_msg) self._data_id += 1 @@ -378,6 +395,9 @@ def insert_steps(self, data_id, step_id, dmaker_type, dmaker_name, data_id_src, def insert_feedback(self, data_id, source, timestamp, content, status_code=None): + if not isinstance(source, FeedbackSource): + source = FeedbackSource(source) + if source not in self.last_feedback: self.last_feedback[source] = [] @@ -389,6 +409,9 @@ def insert_feedback(self, data_id, source, timestamp, content, status_code=None) } ) + if self.current_project: + self.current_project.trigger_feedback_handlers(source, timestamp, content, status_code) + if not self.enabled: return None @@ -397,18 +420,19 @@ def insert_feedback(self, data_id, source, timestamp, content, status_code=None) stmt = "INSERT INTO FEEDBACK(DATA_ID,SOURCE,DATE,CONTENT,STATUS)"\ " VALUES(?,?,?,?,?)" - params = (data_id, source, timestamp, content, status_code) + params = (data_id, str(source), timestamp, content, status_code) err_msg = 'while inserting a value into table FEEDBACK!' self.submit_sql_stmt(stmt, params=params, error_msg=err_msg) def iter_last_feedback_entries(self, source=None): + last_fbk = self.last_feedback.items() if source is None: - for source, fbks in self.last_feedback.items(): + for src, fbks in last_fbk: for item in fbks: status = item['status'] ts = item['timestamp'] content = item['content'] - yield source, status, ts, content + yield src, status, ts, content else: for item in self.last_feedback[source]: status = item['status'] @@ -437,6 +461,15 @@ def insert_fmk_info(self, data_id, content, date, error=False): err_msg = 'while inserting a value into table FMKINFO!' self.submit_sql_stmt(stmt, params=params, error_msg=err_msg) + def insert_analysis(self, data_id, content, date, impact=False): + if not self.enabled: + return None + + stmt = "INSERT INTO ANALYSIS(DATA_ID,CONTENT,DATE,IMPACT)"\ + " VALUES(?,?,?,?)" + params = (data_id, content, date, impact) + err_msg = 'while inserting a value into table ANALYSIS!' + self.submit_sql_stmt(stmt, params=params, error_msg=err_msg) def fetch_data(self, start_id=1, end_id=-1): ign_end_id = '--' if end_id < 1 else '' @@ -481,6 +514,7 @@ def check_data_existence(self, data_id, colorized=True): return data def display_data_info(self, data_id, with_data=False, with_fbk=False, with_fmkinfo=True, + with_analysis=True, fbk_src=None, limit_data_sz=None, page_width=100, colorized=True, raw=False): @@ -533,6 +567,12 @@ def display_data_info(self, data_id, with_data=False, with_fbk=False, with_fmkin "ORDER BY ERROR DESC;".format(data_id=data_id) ) + analysis_records = self.execute_sql_statement( + "SELECT CONTENT, DATE, IMPACT FROM ANALYSIS " + "WHERE DATA_ID == {data_id:d} " + "ORDER BY DATE ASC;".format(data_id=data_id) + ) + line_pattern = '-' * page_width data_id_pattern = " Data ID #{:d} ".format(data_id) @@ -560,8 +600,8 @@ def display_data_info(self, data_id, with_data=False, with_fbk=False, with_fmkin msg += colorize(",\n".format(src) + ' '*len(status_prefix), rgb=Color.FMKINFO) msg += '\n' - sentd = sent_date.strftime("%d/%m/%Y - %H:%M:%S") if sent_date else 'None' - ackd = ack_date.strftime("%d/%m/%Y - %H:%M:%S") if ack_date else 'None' + sentd = sent_date.strftime("%d/%m/%Y - %H:%M:%S.%f") if sent_date else 'None' + ackd = ack_date.strftime("%d/%m/%Y - %H:%M:%S.%f") if ack_date else 'None' msg += colorize(" Sent: ", rgb=Color.FMKINFO) + colorize(sentd, rgb=Color.DATE) msg += colorize("\n Received: ", rgb=Color.FMKINFO) + colorize(ackd, rgb=Color.DATE) msg += colorize("\n Size: ", rgb=Color.FMKINFO) + colorize(str(size) + ' Bytes', @@ -571,7 +611,7 @@ def display_data_info(self, data_id, with_data=False, with_fbk=False, with_fmkin prt(msg) - def handle_dmaker(dmk_pattern, info, dmk_type, dmk_name, name_sep_sz, id_src=None): + def handle_dmaker(dmk_pattern, info, dmk_type, dmk_name, name_sep_sz, ui, id_src=None): msg = '' msg += colorize("\n |_ {:s}: ".format(dmk_pattern), rgb=Color.FMKINFO) msg += colorize(str(dmk_type).ljust(name_sep_sz, ' '), rgb=Color.FMKSUBINFO) @@ -614,21 +654,42 @@ def handle_dmaker(dmk_pattern, info, dmk_type, dmk_name, name_sep_sz, id_src=Non if dmk_type != data_type: msg += colorize("\n |_ Generator: ", rgb=Color.FMKINFO) msg += colorize(str(data_type), rgb=Color.FMKSUBINFO) - msg += colorize(" | UI: ", rgb=Color.FMKINFO) - msg += colorize(str(ui), rgb=Color.FMKSUBINFO) sid += 1 msg += colorize("\n Step #{:d}:".format(sid), rgb=Color.FMKINFOGROUP) - msg += handle_dmaker('Disruptor', info, dmk_type, dmk_name, len(data_type)) + msg += handle_dmaker('Disruptor', info, dmk_type, dmk_name, len(data_type), ui) else: - msg += handle_dmaker('Generator', info, dmk_type, dmk_name, name_sep_sz, + msg += handle_dmaker('Generator', info, dmk_type, dmk_name, name_sep_sz, ui, id_src=id_src) else: msg += colorize("\n Step #{:d}:".format(sid), rgb=Color.FMKINFOGROUP) - msg += handle_dmaker('Disruptor', info, dmk_type, dmk_name, name_sep_sz) + msg += handle_dmaker('Disruptor', info, dmk_type, dmk_name, name_sep_sz, ui) sid += 1 msg += colorize('\n' + line_pattern, rgb=Color.NEWLOGENTRY) prt(msg) + msg = '' + for idx, info in enumerate(analysis_records, start=1): + content, tstamp, impact = info + if not with_analysis: + continue + date_str = tstamp.strftime("%d/%m/%Y - %H:%M") if tstamp else 'Not Dated' + msg += colorize("\n User Analysis: ", rgb=Color.FMKINFOGROUP) + msg += colorize(date_str, rgb=Color.DATE) + msg += colorize(" | ", rgb=Color.FMKINFOGROUP) + if impact: + msg += colorize("Data triggered an unexpected behavior", rgb=Color.ANALYSIS_IMPACT) + else: + msg += colorize("Data did not trigger an unexpected behavior", rgb=Color.ANALYSIS_NO_IMPACT) + if content: + chks = chunk_lines(content, page_width - 10) + for c in chks: + color = Color.FMKHLIGHT if impact else Color.DATAINFO_ALT + msg += '\n' + colorize(' ' * 2 + '| ', rgb=Color.FMKINFOGROUP) + \ + colorize(str(c), rgb=color) + if msg: + msg += colorize('\n' + line_pattern, rgb=Color.NEWLOGENTRY) + prt(msg) + msg = '' for idx, com in enumerate(comments, start=1): content, tstamp = com @@ -674,7 +735,7 @@ def handle_dmaker(dmk_pattern, info, dmk_type, dmk_name, name_sep_sz, id_src=Non if with_fbk: for src, tstamp, status, content in feedback: - formatted_ts = None if tstamp is None else tstamp.strftime("%d/%m/%Y - %H:%M:%S") + formatted_ts = None if tstamp is None else tstamp.strftime("%d/%m/%Y - %H:%M:%S.%f") msg += colorize("\n Status(", rgb=Color.FMKINFOGROUP) msg += colorize("{!s}".format(src), rgb=Color.FMKSUBINFO) msg += colorize(" | ", rgb=Color.FMKINFOGROUP) @@ -703,9 +764,9 @@ def _handle_binary_content(self, content, sz_limit=None, raw=False, colorized=Tr colorize = self._get_color_function(colorized) if sys.version_info[0] > 2: - content = content if raw else '{!a}'.format(content) + content = content if not raw else '{!a}'.format(content) else: - content = content if raw else repr(content) + content = content if not raw else repr(content) if sz_limit is not None and len(content) > sz_limit: content = content[:sz_limit] @@ -715,6 +776,7 @@ def _handle_binary_content(self, content, sz_limit=None, raw=False, colorized=Tr def display_data_info_by_date(self, start, end, with_data=False, with_fbk=False, with_fmkinfo=True, + with_analysis=True, fbk_src=None, prj_name=None, limit_data_sz=None, raw=False, page_width=100, colorized=True): colorize = self._get_color_function(colorized) @@ -736,7 +798,9 @@ def display_data_info_by_date(self, start, end, with_data=False, with_fbk=False, for rec in records: data_id = rec[0] self.display_data_info(data_id, with_data=with_data, with_fbk=with_fbk, - with_fmkinfo=with_fmkinfo, fbk_src=fbk_src, + with_fmkinfo=with_fmkinfo, + with_analysis=with_analysis, + fbk_src=fbk_src, limit_data_sz=limit_data_sz, raw=raw, page_width=page_width, colorized=colorized) else: @@ -744,6 +808,7 @@ def display_data_info_by_date(self, start, end, with_data=False, with_fbk=False, rgb=Color.ERROR)) def display_data_info_by_range(self, first_id, last_id, with_data=False, with_fbk=False, with_fmkinfo=True, + with_analysis=True, fbk_src=None, prj_name=None, limit_data_sz=None, raw=False, page_width=100, colorized=True): @@ -766,7 +831,8 @@ def display_data_info_by_range(self, first_id, last_id, with_data=False, with_fb for rec in records: data_id = rec[0] self.display_data_info(data_id, with_data=with_data, with_fbk=with_fbk, - with_fmkinfo=with_fmkinfo, fbk_src=fbk_src, + with_fmkinfo=with_fmkinfo, with_analysis=with_analysis, + fbk_src=fbk_src, limit_data_sz=limit_data_sz, raw=raw, page_width=page_width, colorized=colorized) else: @@ -921,6 +987,7 @@ def get_project_record(self, prj_name=None): return prj_records def get_data_with_impact(self, prj_name=None, fbk_src=None, display=True, verbose=False, + raw_analysis=False, colorized=True): colorize = self._get_color_function(colorized) @@ -937,6 +1004,16 @@ def get_data_with_impact(self, prj_name=None, fbk_src=None, display=True, verbos "WHERE STATUS < 0;" ) + + analysis_records = self.execute_sql_statement( + "SELECT DATA_ID, CONTENT, DATE, IMPACT FROM ANALYSIS " + "ORDER BY DATE DESC;" + ) + if analysis_records: + data_analyzed = set([x[0] for x in analysis_records]) + else: + data_analyzed = set() + prj_records = self.get_project_record(prj_name) data_list = [] @@ -950,6 +1027,15 @@ def get_data_with_impact(self, prj_name=None, fbk_src=None, display=True, verbos id2fbk[data_id][src] = [] id2fbk[data_id][src].append(status) + user_src = 'User Analysis' + for rec in analysis_records: + data_id, content, tstamp, impact = rec + if data_id not in id2fbk: + id2fbk[data_id] = {} + if user_src not in id2fbk[data_id]: + id2fbk[data_id][user_src] = [] + id2fbk[data_id][user_src].append(impact) + data_id_pattern = "{:>" + str(int(math.log10(len(prj_records))) + 2) + "s}" format_string = " [DataID " + data_id_pattern + "] --> {:s}" @@ -957,6 +1043,16 @@ def get_data_with_impact(self, prj_name=None, fbk_src=None, display=True, verbos for rec in prj_records: data_id, target, prj = rec if data_id in id2fbk: + + if not raw_analysis and data_id in data_analyzed: + records = self.execute_sql_statement( + "SELECT IMPACT FROM ANALYSIS " + "WHERE DATA_ID == {data_id:d} " + "ORDER BY DATE DESC;".format(data_id=data_id) + ) + if records[0][0] == False: + continue + data_list.append(data_id) if display: if prj != current_prj: @@ -967,10 +1063,19 @@ def get_data_with_impact(self, prj_name=None, fbk_src=None, display=True, verbos rgb=Color.DATAINFO)) if verbose: for src, status in id2fbk[data_id].items(): - status_str = ''.join([str(s) + ',' for s in status])[:-1] - print(colorize(" |_ status={:s} from {:s}".format(status_str, - src), - rgb=Color.FMKSUBINFO)) + if src == user_src: + if status[0] == 0: + status_str = 'User analysis carried out: False Positive' + else: + status_str = 'User analysis carried out: Impact Confirmed' + color = Color.ANALYSIS_FALSEPOSITIVE if status[0] == 0 else Color.ANALYSIS_CONFIRM + print(colorize(" |_ {:s}".format(status_str), + rgb=color)) + else: + status_str = ''.join([str(s) + ',' for s in status])[:-1] + print(colorize(" |_ status={:s} from {:s}" + .format(status_str, src), + rgb=Color.FMKSUBINFO)) else: print(colorize("*** No data has negatively impacted a target ***", rgb=Color.FMKINFO)) diff --git a/framework/dmhelpers/generic.py b/framework/dmhelpers/generic.py index 97e907a..d7a3f49 100644 --- a/framework/dmhelpers/generic.py +++ b/framework/dmhelpers/generic.py @@ -236,7 +236,8 @@ def timestamp(time_format, utc, set_attrs, clear_attrs): def CRC(vt=fvt.INT_str, poly=0x104c11db7, init_crc=0, xor_out=0xFFFFFFFF, rev=True, - set_attrs=None, clear_attrs=None, after_encoding=True, freezable=False): + set_attrs=None, clear_attrs=None, after_encoding=True, freezable=False, + base=16, letter_case='upper', reverse_str=False): """ Return a *generator* that returns the CRC (in the chosen type) of all the node parameters. (Default CRC is PKZIP CRC32) @@ -253,11 +254,16 @@ def CRC(vt=fvt.INT_str, poly=0x104c11db7, init_crc=0, xor_out=0xFFFFFFFF, rev=Tr set to False only if node arguments support encoding. freezable (bool): if ``False`` make the generator unfreezable in order to always provide the right value. (Note that tTYPE will still be able to corrupt the generator.) + base (int): Relevant when ``vt`` is ``INT_str``. Numerical base to use for string representation + letter_case (str): Relevant when ``vt`` is ``INT_str``. Letter case for string representation + ('upper' or 'lower') + reverse_str (bool): Reverse the order of the string if set to ``True``. """ class Crc(object): unfreezable = not freezable - def __init__(self, vt, poly, init_crc, xor_out, rev, set_attrs, clear_attrs): + def __init__(self, vt, poly, init_crc, xor_out, rev, set_attrs, clear_attrs, + base=16, letter_case='upper', reverse_str=False): self.vt = vt self.poly = poly self.init_crc = init_crc @@ -265,6 +271,9 @@ def __init__(self, vt, poly, init_crc, xor_out, rev, set_attrs, clear_attrs): self.rev = rev self.set_attrs = set_attrs self.clear_attrs = clear_attrs + self.base = base + self.letter_case = letter_case + self.reverse_str = reverse_str def __call__(self, nodes): crc_func = crcmod.mkCrcFun(self.poly, initCrc=self.init_crc, @@ -283,7 +292,12 @@ def __call__(self, nodes): result = crc_func(s) - n = Node('cts', value_type=self.vt(values=[result], force_mode=True)) + if issubclass(self.vt, fvt.INT_str): + n = Node('cts', value_type=self.vt(values=[result], force_mode=True, + base=self.base, letter_case=self.letter_case, + reverse=self.reverse_str)) + else: + n = Node('cts', value_type=self.vt(values=[result], force_mode=True)) n.set_semantics(NodeSemantics(['crc'])) MH._handle_attrs(n, self.set_attrs, self.clear_attrs) return n @@ -292,7 +306,8 @@ def __call__(self, nodes): raise NotImplementedError('the CRC template has been disabled because python-crcmod module is not installed!') vt = MH._validate_int_vt(vt) - return Crc(vt, poly, init_crc, xor_out, rev, set_attrs, clear_attrs) + return Crc(vt, poly, init_crc, xor_out, rev, set_attrs, clear_attrs, + base=base, letter_case=letter_case, reverse_str=reverse_str) diff --git a/framework/dmhelpers/json.py b/framework/dmhelpers/json.py new file mode 100755 index 0000000..26736e6 --- /dev/null +++ b/framework/dmhelpers/json.py @@ -0,0 +1,149 @@ +################################################################################ +# +# Copyright 2017 Rockwell Collins Inc. +# +################################################################################ +# +# This file is part of fuddly. +# +# fuddly is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# fuddly is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with fuddly. If not, see +# +################################################################################ + +from framework.node import * +from framework.dmhelpers.generic import * +import framework.value_types as fvt +import framework.global_resources as gr +import uuid + +def json_builder(tag_name, sample=None, node_name=None, codec='latin-1', + tag_name_mutable=True, struct_mutable=True, determinist=True): + """ + Helper for modeling an JSON structure. + + Args: + tag_name (str): name of the JSON tag. + sample (dict): the JSON structure to be converted to a fuddly structure + node_name (str): name of the node to be created. + codec (str): codec to be used for generating the JSON structure. + tag_name_mutable (bool): if ``False``, the tag name will not be mutable, meaning that + its ``Mutable`` attribute will be cleared. + struct_mutable (bool): if ``False`` the JSON structure "will not" be mutable, meaning + that each node related to the structure will have its ``Mutable`` attribute cleared. + determinist (bool): if ``False``, the attribute order could change from one retrieved + data to another. + + Returns: + dict: Node-description of the JSON structure. + """ + + if sample is not None: + assert isinstance(sample, dict) + cts = [] + idx = 1 + for k, v in sample.items(): + sep_id = uuid.uuid1() # The separator for the " in the key param. e.g., "" + + params = [ + {'name': ('sep', sep_id), 'contents' : fvt.String(values=['"'], codec=codec), + 'set_attrs': MH.Attr.Separator, 'mutable': struct_mutable}, + {'name': ('key', uuid.uuid1()), 'contents' : fvt.String(values=[k], codec=codec)}, + {'name': ('sep', sep_id)}, + {'name': ('col', uuid.uuid1()), 'contents' : fvt.String(values=[':'], codec=codec), + 'set_attrs': MH.Attr.Separator, 'mutable': struct_mutable} ] + + if isinstance(v, list): + modeled_v = [] + val_id = uuid.uuid1() + for subidx, value in enumerate(v): + assert not isinstance(value, list) + if isinstance(value, dict): + # If the type of v is a dictionary, build a sub JSON structure for it. + modeled_v.append(json_builder(tag_name + "_" + str(idx)+str(subidx), sample=value)) + else: + checked_value = value if gr.is_string_compatible(value) else str(value) + modeled_v.append( + {'name': ('val'+str(subidx), val_id), + 'contents': [ + {'name': ('sep', sep_id)}, + {'name': ('cts', uuid.uuid1()), + 'contents': fvt.String(values=[checked_value], codec=codec)}, + {'name': ('sep', sep_id)} ]} + ) + + attr_value = \ + {'name': ('cts', uuid.uuid1()), + 'contents': modeled_v, + 'separator': {'contents': {'name': ('comma', uuid.uuid1()), + 'contents': fvt.String(values=[','], max_sz=100, + absorb_regexp='\s*,\s*', codec=codec), + 'mutable': struct_mutable, + 'absorb_csts': AbsNoCsts(size=True, regexp=True)}, + 'prefix': False, 'suffix': False, 'unique': False} } + + params.append({'name': ('attr_val'+str(idx), uuid.uuid1()), + 'contents': [ + {'contents': fvt.String(values=['['], codec=codec), + 'mutable': struct_mutable, 'name': 'prefix'+str(idx)}, + attr_value, + {'contents': fvt.String(values=[']'], codec=codec), + 'mutable': struct_mutable, 'name': 'suffix'+str(idx)} ]}) + + elif isinstance(v, dict): + params.append(json_builder(tag_name + "_" + str(idx), sample=v)) + + elif gr.is_string_compatible(v): + params += [ {'name': ('sep', sep_id)}, + {'name': ('val', uuid.uuid1()), 'contents': fvt.String(values=[v], codec=codec)}, + {'name': ('sep', sep_id)} ] + else: + raise DataModelDefinitionError + + cts.append({'name': ('attr'+str(idx), uuid.uuid1()), + 'contents': params}) + idx += 1 + + if not determinist: + params_desc = {'section_type': MH.FullyRandom, 'contents': cts} + else: + params_desc = {'section_type': MH.Ordered, 'contents': cts} + else: + raise DataModelDefinitionError + + tag_start_open_desc = \ + {'name': ('prefix', uuid.uuid1()), + 'contents': fvt.String(values=['{'], codec=codec), + 'mutable': struct_mutable, 'set_attrs': MH.Attr.Separator} + + tag_start_cts_desc = \ + {'name': ('contents', uuid.uuid1()), + 'random': not determinist, + 'separator': {'contents': {'name': ('comma', uuid.uuid1()), + 'contents': fvt.String(values=[','], max_sz=100, + absorb_regexp='\s*,\s*', codec=codec), + 'mutable': struct_mutable, + 'absorb_csts': AbsNoCsts(size=True, regexp=True)}, + 'prefix': False, 'suffix': False, 'unique': False}, + 'contents': [params_desc]} + + tag_start_close_desc = \ + {'name': ('suffix', uuid.uuid1()), + 'contents': fvt.String(values=['}'], codec=codec), + 'mutable': struct_mutable, 'set_attrs': MH.Attr.Separator} + + tag_start_desc = \ + {'name': tag_name if node_name is None else node_name, + 'contents': [tag_start_open_desc, tag_start_cts_desc, tag_start_close_desc]} + + return tag_start_desc diff --git a/framework/dmhelpers/xml.py b/framework/dmhelpers/xml.py index 162902f..105ccb6 100644 --- a/framework/dmhelpers/xml.py +++ b/framework/dmhelpers/xml.py @@ -25,15 +25,26 @@ from framework.dmhelpers.generic import * import framework.value_types as fvt import framework.global_resources as gr +from enum import Enum -def tag_builder(tag_name, params=None, contents=None, node_name=None, codec='latin-1', - tag_name_mutable=True, struct_mutable=True, determinist=True): +class TAG_TYPE(Enum): + standard = 1 + comment = 2 + proc_instr = 3 + +def tag_builder(tag_name, params=None, refs=None, contents=None, node_name=None, codec='latin-1', + tag_name_mutable=True, struct_mutable=True, determinist=True, condition=None, + absorb_regexp=None, specific_fuzzy_vals=None, + tag_type=TAG_TYPE.standard, nl_prefix=False, nl_suffix=False): """ Helper for modeling an XML tag. Args: tag_name (str): name of the XML tag. params (dict): optional attributes to be added in the XML tag + refs (dict): if provided it should contain for at least one parameter key (provided in ``params`` dict) + the name to be used for the node representing the corresponding value. Useful when + the parameter ``condition`` is in use and needs to relate to the value of specific parameters. contents: can be either None (empty tag), a :class:`framework.data_model.Node`, a dictionary (Node description), a string or a string list (string-Node values). node_name (str): name of the node to be created. @@ -44,6 +55,15 @@ def tag_builder(tag_name, params=None, contents=None, node_name=None, codec='lat that each node related to the structure will have its ``Mutable`` attribute cleared. determinist (bool): if ``False``, the attribute order could change from one retrieved data to another. + condition (tuple): optional existence condition for the tag. If not ``None`` a keyword ``exists_if`` + will be added to the root node with this parameter as a value. + absorb_regexp (str): regex for ``contents`` absorption + tag_type (TAG_TYPE): specify the type of notation + specific_fuzzy_vals (dict): if provided it should contain for at least one parameter key (provided + in ``params`` dict) a list of specific values that will be used by some generic disruptors + like tTYPE. + nl_prefix (bool): add a new line character before the tag + nl_suffix (bool): add a new line character after the tag Returns: dict: Node-description of the XML tag. @@ -53,9 +73,24 @@ def tag_builder(tag_name, params=None, contents=None, node_name=None, codec='lat assert isinstance(params, dict) cts = [] idx = 1 + refs = {} if refs is None else refs for k, v in params.items(): - assert gr.is_string_compatible(v) - v = v if isinstance(v, list) else [v] + if gr.is_string_compatible(v): + val_ref = refs.get(k, ('val', uuid.uuid1())) + v = v if isinstance(v, list) else [v] + nd_desc = {'name': val_ref, 'contents': fvt.String(values=v, codec=codec)} + if specific_fuzzy_vals: + nd_desc['specific_fuzzy_vals'] = specific_fuzzy_vals.get(k) + elif isinstance(v, dict): + nd_desc = v + if specific_fuzzy_vals: + nd_desc['specific_fuzzy_vals'] = specific_fuzzy_vals.get(k) + elif isinstance(v, Node): + nd_desc = (v, 1, 1) + v.set_specific_fuzzy_vals(specific_fuzzy_vals.get(k)) + else: + raise ValueError + sep_id = uuid.uuid1() cts.append({'name': ('attr'+str(idx), uuid.uuid1()), 'contents': [ @@ -64,7 +99,7 @@ def tag_builder(tag_name, params=None, contents=None, node_name=None, codec='lat 'set_attrs': MH.Attr.Separator, 'mutable': struct_mutable}, {'name': ('sep', sep_id), 'contents': fvt.String(values=['"'], codec=codec), 'set_attrs': MH.Attr.Separator, 'mutable': struct_mutable}, - {'name': ('val', uuid.uuid1()), 'contents': fvt.String(values=v, codec=codec)}, + nd_desc, {'name': ('sep', sep_id)}, ]}) idx += 1 @@ -76,7 +111,17 @@ def tag_builder(tag_name, params=None, contents=None, node_name=None, codec='lat else: params_desc = None - prefix = '' + elif tag_type is TAG_TYPE.comment: + suffix = '-->' + else: + raise ValueError + else: + suffix = '>' + tag_start_close_desc = \ {'name': ('suffix', uuid.uuid1()), - 'contents': fvt.String(values=['>'], codec=codec), + 'contents': fvt.String(values=[suffix], codec=codec), 'mutable': struct_mutable, 'set_attrs': MH.Attr.Separator} + if contents is None: + start_tag_name = tag_name + else: + start_tag_name = ('start-tag', uuid.uuid1()) + tag_start_desc = \ - {'name': ('start-tag', uuid.uuid1()), + {'name': start_tag_name, 'contents': [tag_start_open_desc, tag_start_cts_desc, tag_start_close_desc]} tag_end_desc = \ @@ -133,22 +193,47 @@ def tag_builder(tag_name, params=None, contents=None, node_name=None, codec='lat cts = [tag_start_desc, contents, tag_end_desc] + elif isinstance(contents, list) and not gr.is_string_compatible(contents[0]): + cts = [tag_start_desc] + for c in contents: + cts.append(c) + cts.append(tag_end_desc) else: assert gr.is_string_compatible(contents) if not isinstance(contents, list): contents = [contents] + content_desc = {'name': ('elt-content', uuid.uuid1()), + 'contents': fvt.String(values=contents, codec=codec, absorb_regexp=absorb_regexp)} + if absorb_regexp is not None: + content_desc['absorb_csts'] = AbsNoCsts(regexp=True) + cts = [tag_start_desc, - {'name': 'elt-content', - 'contents': fvt.String(values=contents, codec=codec)}, + content_desc, tag_end_desc] tag_desc = \ {'name': tag_name if node_name is None else node_name, 'separator': {'contents': {'name': ('nl', uuid.uuid1()), 'contents': fvt.String(values=['\n'], max_sz=100, - absorb_regexp='[\r\n|\n]+', codec=codec), + absorb_regexp='\s*', codec=codec), 'absorb_csts': AbsNoCsts(regexp=True)}, - 'prefix': False, 'suffix': False, 'unique': False}, + 'prefix': nl_prefix, 'suffix': nl_suffix, 'unique': False}, 'contents': cts} + if condition: + tag_desc['exists_if'] = condition + return tag_desc + +def xml_decl_builder(determinist=True): + version_desc = {'name': 'version', + 'contents': '[123456789]\.\d'} + + encoding_list = ['UTF-8', 'UTF-16', 'ISO-10646-UCS-2','ISO-10646-UCS-4', + 'ISO-2022-JP', 'Shift_JIS', 'EUC-JP'] + \ + ['ISO-8859-{:d}'.format(x) for x in range(1, 10)] + + return tag_builder('xml', params={'version': version_desc, + 'encoding': encoding_list, + 'standalone': ['no', 'yes']}, + tag_type=TAG_TYPE.proc_instr, determinist=determinist) diff --git a/framework/error_handling.py b/framework/error_handling.py index feffd39..06afd97 100644 --- a/framework/error_handling.py +++ b/framework/error_handling.py @@ -32,6 +32,7 @@ class PopulationError(Exception): pass class ExtinctPopulationError(PopulationError): pass class DataModelDefinitionError(Exception): pass +class ProjectDefinitionError(Exception): pass class RegexParserError(DataModelDefinitionError): pass diff --git a/framework/evolutionary_helpers.py b/framework/evolutionary_helpers.py index 46b5424..0bb74e3 100644 --- a/framework/evolutionary_helpers.py +++ b/framework/evolutionary_helpers.py @@ -103,7 +103,7 @@ def __init__(self, fmk, node): self.probability_of_survival = None # between 0 and 1 def mutate(self, nb): - data = self._fmk.get_data([('C', None, UI(nb=nb))], data_orig=Data(self.node)) + data = self._fmk.get_data([('C', UI(nb=nb))], data_orig=Data(self.node)) if data is None: raise PopulationError assert isinstance(data.content, Node) @@ -188,7 +188,7 @@ def _crossover(self): while True: - data = self._fmk.get_data([('tCOMB', None, UI(node=ind_2))], data_orig=Data(ind_1)) + data = self._fmk.get_data([('tCOMB', UI(node=ind_2))], data_orig=Data(ind_1)) if data is None or data.is_unusable(): break else: @@ -241,7 +241,7 @@ def cbk_after(env, current_step, next_step, fbk): return True - step = Step(data_desc=DataProcess(process=[('POPULATION', None, UI(population=population))])) + step = Step(data_desc=DataProcess(process=[('POPULATION', UI(population=population))])) step.connect_to(step, cbk_after_fbk=cbk_after) return Scenario(name, anchor=step) diff --git a/framework/fmk_db.sql b/framework/fmk_db.sql index 9d1a712..e8c0452 100644 --- a/framework/fmk_db.sql +++ b/framework/fmk_db.sql @@ -99,6 +99,14 @@ CREATE TABLE FMKINFO ( ERROR BOOLEAN ); +CREATE TABLE ANALYSIS ( + ID INTEGER PRIMARY KEY ASC AUTOINCREMENT, + DATA_ID INTEGER REFERENCES DATA (ID), + CONTENT TEXT, + DATE TIMESTAMP, + IMPACT BOOLEAN +); + CREATE VIEW STATS AS SELECT TYPE, sum(CPT) as TOTAL FROM ( diff --git a/framework/fuzzing_primitives.py b/framework/fuzzing_primitives.py index 2f37139..61162bd 100644 --- a/framework/fuzzing_primitives.py +++ b/framework/fuzzing_primitives.py @@ -84,6 +84,7 @@ def __init__(self, root_node, node_consumer, make_determinist=False, make_random def set_consumer(self, node_consumer): self._consumer = node_consumer self._consumer._root_node = self._root_node + self._consumer.preload(self._root_node) def __iter__(self): @@ -336,7 +337,6 @@ def _do_if_not_interested(node, orig_node_val): consume_called_again = True - node.get_value() not_recovered = True else: if node in consumed_nodes: @@ -349,14 +349,43 @@ def _do_if_not_interested(node, orig_node_val): else: yield node, orig_node_val, False, False - if max_steps != 0 and not consume_called_again: + # if max_steps != 0 and not consume_called_again: + # max_steps -= 1 + # # In this case we iterate only on the current node + # node.unfreeze(recursive=False, ignore_entanglement=True) + # node.freeze() + # if self._consumer.fix_constraints: + # node.fix_synchronized_nodes() + # elif not consume_called_again: + # if not_recovered and (self._consumer.interested_by(node) or node in consumed_nodes): + # self._consumer.recover_node(node) + # if self._consumer.fix_constraints: + # node.fix_synchronized_nodes() + # if not node.is_exhausted() and self._consumer.need_reset(node): + # yield None, None, True, True + # again = False + # + # else: + # consume_called_again = False + # node.freeze() + + if max_steps != 0: max_steps -= 1 - # In this case we iterate only on the current node - node.unfreeze(recursive=False, ignore_entanglement=True) + if consume_called_again: + node.freeze() + consume_called_again = False + else: + # In this case we iterate only on the current node + node.unfreeze(recursive=False, ignore_entanglement=True) + node.freeze() + if self._consumer.fix_constraints: + node.fix_synchronized_nodes() + + elif consume_called_again: node.freeze() - if self._consumer.fix_constraints: - node.fix_synchronized_nodes() - elif not consume_called_again: + consume_called_again = False + + else: if not_recovered and (self._consumer.interested_by(node) or node in consumed_nodes): self._consumer.recover_node(node) if self._consumer.fix_constraints: @@ -365,8 +394,6 @@ def _do_if_not_interested(node, orig_node_val): yield None, None, True, True again = False - else: - consume_called_again = False return @@ -401,6 +428,17 @@ def __init__(self, max_runs_per_node=-1, min_runs_per_node=-1, respect_order=Tru def init_specific(self, **kwargs): self._internals_criteria = dm.NodeInternalsCriteria(negative_node_kinds=[dm.NodeInternals_NonTerm]) + def preload(self, root_node): + """ + Called by the ModelWalker when it initializes + + Args: + root_node: Root node of the modeled data + + Returns: None + + """ + pass def consume_node(self, node): ''' @@ -522,7 +560,7 @@ def interested_by(self, node): class BasicVisitor(NodeConsumerStub): - def init_specific(self): + def init_specific(self, **kwargs): self._internals_criteria = dm.NodeInternalsCriteria(negative_node_kinds=[dm.NodeInternals_NonTerm]) self.firstcall = True @@ -698,7 +736,7 @@ def wait_for_exhaustion(self, node): class TypedNodeDisruption(NodeConsumerStub): - def init_specific(self, ignore_separator=False): + def init_specific(self, ignore_separator=False, enforce_determinism=True): if ignore_separator: self._internals_criteria = dm.NodeInternalsCriteria(mandatory_attrs=[dm.NodeInternals.Mutable], negative_attrs=[dm.NodeInternals.Separator], @@ -713,9 +751,18 @@ def init_specific(self, ignore_separator=False): self.current_fuzz_vt_list = None self.current_node = None self.orig_internal = None + self.enforce_determinism = enforce_determinism + self._ignore_separator = ignore_separator + self.sep_list = None self.need_reset_when_structure_change = True + def preload(self, root_node): + if not self._ignore_separator: + ic = dm.NodeInternalsCriteria(mandatory_attrs=[dm.NodeInternals.Separator]) + self.sep_list = set(map(lambda x: x.to_bytes(), root_node.get_reachable_nodes(internals_criteria=ic))) + self.sep_list = list(self.sep_list) + def consume_node(self, node): if node.is_genfunc() and (node.is_attr_set(dm.NodeInternals.Freezable) or not node.generated_node.is_typed_value()): @@ -730,10 +777,8 @@ def consume_node(self, node): self.orig_all_attrs = node.cc.get_attrs_copy() # self.orig_value = node.to_bytes() - vt_node = node.generated_node if node.is_genfunc() else node - self.current_fuzz_vt_list = self._create_fuzzy_vt_list(vt_node, self.fuzz_magnitude) - self._extend_fuzzy_vt_list(self.current_fuzz_vt_list, vt_node) + self._populate_fuzzy_vt_list(vt_node, self.fuzz_magnitude) DEBUG_PRINT(' *** CONSUME: ' + node.name + ', ' + repr(self.current_fuzz_vt_list), level=0) @@ -742,7 +787,8 @@ def consume_node(self, node): node.set_values(value_type=vt_obj, ignore_entanglement=True, preserve_node=True) node.make_finite() - node.make_determinist() + if self.enforce_determinism: + node.make_determinist() node.unfreeze(ignore_entanglement=True) # we need to be sure that the current node is freezable node.set_attr(dm.NodeInternals.Freezable) @@ -752,6 +798,44 @@ def consume_node(self, node): else: raise ValueError + def _populate_fuzzy_vt_list(self, vt_node, fuzz_magnitude): + + vt = vt_node.get_value_type() + + if issubclass(vt.__class__, vtype.VT_Alt): + new_vt = copy.copy(vt) + new_vt.make_private(forget_current_state=False) + new_vt.enable_fuzz_mode(fuzz_magnitude=fuzz_magnitude) + self.current_fuzz_vt_list = [new_vt] + else: + self.current_fuzz_vt_list = [] + + fuzzed_vt = vt.get_fuzzed_vt_list() + if fuzzed_vt: + self.current_fuzz_vt_list += fuzzed_vt + + if self.sep_list: + self._add_separator_cases(vt_node) + + def _add_separator_cases(self, vt_node): + current_val = vt_node.get_current_value() + if vt_node.is_attr_set(dm.NodeInternals.Separator): + sep_l = copy.copy(self.sep_list) + try: + sep_l.remove(current_val) + except ValueError: + print("\n*** WARNING: separator not part of the initial set. (Could happen if " + "separators are generated dynamically)") + self.current_fuzz_vt_list.insert(0, vtype.String(values=sep_l)) + else: + sz = len(current_val) + if sz > 1: + fuzzy_sep_val_list = [] + for sep in self.sep_list: + new_val = current_val[:-1] + sep + current_val[-1:] + fuzzy_sep_val_list.append(new_val) + self.current_fuzz_vt_list.insert(0, vtype.String(values=fuzzy_sep_val_list)) + def save_node(self, node): pass @@ -771,92 +855,6 @@ def still_interested_by(self, node): else: return False - @staticmethod - def _create_fuzzy_vt_list(e, fuzz_magnitude): - vt = e.cc.get_value_type() - - if issubclass(vt.__class__, vtype.VT_Alt): - new_vt = copy.copy(vt) - new_vt.make_private(forget_current_state=False) - new_vt.enable_fuzz_mode(fuzz_magnitude=fuzz_magnitude) - fuzzy_vt_list = [new_vt] - - else: - fuzzy_vt_cls = list(vt.fuzzy_cls.values()) - fuzzy_vt_list = [] - for c in fuzzy_vt_cls: - fuzzy_vt_list.append(c(vt.endian)) - - return fuzzy_vt_list - - @staticmethod - def _extend_fuzzy_vt_list(flist, e): - vt = e.cc.get_value_type() - - if issubclass(vt.__class__, vtype.VT_Alt): - return - - specific_fuzzy_vals = e.cc.get_specific_fuzzy_values() - - val = vt.get_current_raw_val() - if val is not None: - # don't use a set to preserve determinism if needed - supp_list = [val + 1, val - 1] - - if vt.values is not None: - orig_set = set(vt.values) - max_oset = max(orig_set) - min_oset = min(orig_set) - if min_oset != max_oset: - diff_sorted = sorted(set(range(min_oset, max_oset+1)) - orig_set) - if diff_sorted: - item1 = diff_sorted[0] - item2 = diff_sorted[-1] - if item1 not in supp_list: - supp_list.append(item1) - if item2 not in supp_list: - supp_list.append(item2) - if max_oset+1 not in supp_list: - supp_list.append(max_oset+1) - if min_oset-1 not in supp_list: - supp_list.append(min_oset-1) - - if vt.mini is not None: - cond1 = False - if hasattr(vt, 'size'): - cond1 = (vt.mini != 0 or vt.maxi != ((1 << vt.size) - 1)) and \ - (vt.mini != -(1 << (vt.size-1)) or vt.maxi != ((1 << (vt.size-1)) - 1)) - else: - cond1 = True - - if cond1: - if vt.mini-1 not in supp_list: - supp_list.append(vt.mini-1) - if vt.maxi+1 not in supp_list: - supp_list.append(vt.maxi+1) - - if specific_fuzzy_vals is not None: - for v in specific_fuzzy_vals: - if v not in supp_list: - supp_list.append(v) - - fuzzy_vt_obj = None - for o in flist: - # We don't need to check with vt.mini-1 or vt.maxi+1, - # as the following test will provide the first - # compliant choice that will also be OK for the - # previous values (ortherwise, no VT will be OK, and - # these values will be filtered through the call to - # extend_value_list()) - if o.is_compatible(val + 1) or o.is_compatible(val - 1): - fuzzy_vt_obj = o - break - - if fuzzy_vt_obj is not None: - fuzzy_vt_obj.extend_value_list(supp_list) - fuzzy_vt_obj.remove_value_list([val]) - - class SeparatorDisruption(NodeConsumerStub): @@ -883,6 +881,8 @@ def consume_node(self, node): # operation, especially usefull in our case, because we have # to preserve dm.NodeInternals.Separator + node.unfreeze() # ignore previous state + node.make_finite() node.make_determinist() diff --git a/framework/generic_data_makers.py b/framework/generic_data_makers.py index 1d786e3..fad3dbe 100755 --- a/framework/generic_data_makers.py +++ b/framework/generic_data_makers.py @@ -21,6 +21,7 @@ # ################################################################################ +import types import subprocess from copy import * @@ -31,6 +32,7 @@ from framework.basic_primitives import * from framework.value_types import * from framework.data_model import DataModel +from framework.dmhelpers.generic import MH # from framework.plumbing import * from framework.evolutionary_helpers import Population @@ -44,7 +46,6 @@ ####################### @generator(tactics, gtype='POPULATION', weight=1, - gen_args=GENERIC_ARGS, args={'population': ('The population to iterate over.', None, Population)}) class g_population(Generator): """ Walk through the given population """ @@ -92,8 +93,7 @@ def truncate_info(info, max_size=60): return repr(info) -@disruptor(tactics, dtype="tWALK", weight=1, - gen_args = GENERIC_ARGS, +@disruptor(tactics, dtype="tWALK", weight=1, modelwalker_user=True, args={'path': ('Graph path regexp to select nodes on which' \ ' the disruptor should apply.', None, str), 'order': ('When set to True, the walking order is strictly guided ' \ @@ -155,8 +155,7 @@ def disrupt_data(self, dm, target, data): -@disruptor(tactics, dtype="tTYPE", weight=1, - gen_args = GENERIC_ARGS, +@disruptor(tactics, dtype="tTYPE", weight=1, modelwalker_user=True, args={'path': ('Graph path regexp to select nodes on which' \ ' the disruptor should apply.', None, str), 'order': ('When set to True, the fuzzing order is strictly guided ' \ @@ -164,20 +163,38 @@ def disrupt_data(self, dm, target, data): 'in the data model) is used for ordering.', True, bool), 'deep': ('When set to True, if a node structure has changed, the modelwalker ' \ 'will reset its walk through the children nodes.', True, bool), - 'ign_sep': ('When set to True, non-terminal separators will be ignored ' \ + 'ign_sep': ('When set to True, separators will be ignored ' \ 'if any are defined.', False, bool), 'fix_all': ('For each produced data, reevaluate the constraints on the whole graph.', False, bool), 'fix': ("Limit constraints fixing to the nodes related to the currently fuzzed one" " (only implemented for 'sync_size_with' and 'sync_enc_size_with').", True, bool), 'fuzz_mag': ('Order of magnitude for maximum size of some fuzzing test cases.', - 1.0, float)}) + 1.0, float), + 'determinism': ("If set to 'True', the whole model will be fuzzed in " + "a deterministic way. Otherwise it will be guided by the " + "data model determinism.", True, bool), + 'leaf_determinism': ("If set to 'True', each typed node will be fuzzed in " + "a deterministic way. Otherwise it will be guided by the " + "data model determinism. Note: this option is complementary to " + "'determinism' is it acts on the typed node substitutions " + "that occur through this disruptor", True, bool), + }) class sd_fuzz_typed_nodes(StatefulDisruptor): - ''' - Perform alterations on typed nodes (one at a time) according to - its type and various complementary information (such as size, - allowed values, ...). - ''' + """ + Perform alterations on typed nodes (one at a time) according to: + - their type (e.g., INT, Strings, ...) + - their attributes (e.g., allowed values, minimum size, ...) + - knowledge retrieved from the data (e.g., if the input data uses separators, their symbols + are leveraged in the fuzzing) + - knowledge on the target retrieved from the project file or dynamically from feedback inspection + (e.g., C language, GNU/Linux OS, ...) + + If the input has different shapes (described in non-terminal nodes), this will be taken into + account by fuzzing every shape combinations. + + Note: this disruptor includes what tSEP does and goes beyond with respect to separators. + """ def setup(self, dm, user_input): return True @@ -187,17 +204,17 @@ def set_seed(self, prev_data): prev_data.add_info('DONT_PROCESS_THIS_KIND_OF_DATA') return prev_data - prev_content.make_finite(all_conf=True, recursive=True) - self.consumer = TypedNodeDisruption(max_runs_per_node=self.max_runs_per_node, min_runs_per_node=self.min_runs_per_node, fuzz_magnitude=self.fuzz_mag, fix_constraints=self.fix, respect_order=self.order, - ignore_separator=self.ign_sep) + ignore_separator=self.ign_sep, + enforce_determinism=self.leaf_determinism) self.consumer.need_reset_when_structure_change = self.deep self.consumer.set_node_interest(path_regexp=self.path) - self.modelwalker = ModelWalker(prev_content, self.consumer, max_steps=self.max_steps, initial_step=self.init) + self.modelwalker = ModelWalker(prev_content, self.consumer, max_steps=self.max_steps, + initial_step=self.init, make_determinist=self.determinism) self.walker = iter(self.modelwalker) self.max_runs = None @@ -242,13 +259,13 @@ def disrupt_data(self, dm, target, data): data.add_info('reevaluate all the constraints (if any)') data.update_from(exported_node) + data.altered = True return data -@disruptor(tactics, dtype="tALT", weight=1, - gen_args = GENERIC_ARGS, +@disruptor(tactics, dtype="tALT", weight=1, modelwalker_user=True, args={'conf': ("Change the configuration, with the one provided (by name), of " \ "all nodes reachable from the root, one-by-one. [default value is set " \ "dynamically with the first-found existing alternate configuration]", @@ -341,8 +358,7 @@ def disrupt_data(self, dm, target, data): return data -@disruptor(tactics, dtype="tSEP", weight=1, - gen_args = GENERIC_ARGS, +@disruptor(tactics, dtype="tSEP", weight=1, modelwalker_user=True, args={'path': ('Graph path regexp to select nodes on which' \ ' the disruptor should apply.', None, str), 'order': ('When set to True, the fuzzing order is strictly guided ' \ @@ -421,27 +437,28 @@ def disrupt_data(self, dm, target, data): else: data.update_from(rnode) + data.altered = True return data @disruptor(tactics, dtype="tSTRUCT", weight=1, - gen_args={'init': ('Make the model walker ignore all the steps until the provided one.', 1, int), - 'max_steps': ('Maximum number of steps (-1 means until the end).', -1, int) }, - args={'path': ('Graph path regexp to select nodes on which' \ + args={'init': ('Make the model walker ignore all the steps until the provided one.', 1, int), + 'max_steps': ('Maximum number of steps (-1 means until the end).', -1, int), + 'path': ('Graph path regexp to select nodes on which' \ ' the disruptor should apply.', None, str), - 'deep': ('If True, enable corruption of minimum and maxium amount of non-terminal nodes.', + 'deep': ('If True, enable corruption of non-terminal node internals', False, bool) }) class sd_struct_constraints(StatefulDisruptor): - ''' - For each node associated to existence constraints or quantity - constraints, alter the constraint, one at a time, after each call - to this disruptor. + """ + Perform constraints alteration (one at a time) on each node that depends on another one + regarding its existence, its quantity, its size, ... - If `deep` is set, enable new structure corruption cases, based on - the minimum and maximum amount of non-terminal nodes (within the - input data) specified in the data model. - ''' + If `deep` is set, enable more corruption cases on the data structure, based on the internals of + each non-terminal node: + - the minimum and maximum amount of the subnodes of each non-terminal nodes + - ... + """ def setup(self, dm, user_input): return True @@ -604,6 +621,7 @@ def disrupt_data(self, dm, target, data): data.add_info(' |_ {:s}'.format(op_performed)) data.update_from(corrupted_seed) + data.altered = True return data @@ -641,11 +659,11 @@ def disrupt_data(self, dm, target, data): data.update_from(self.node) self.count += 1 + data.altered = True return data @disruptor(tactics, dtype="tCROSS", weight=1, - gen_args=GENERIC_ARGS, args={'node': ('Node to crossover with.', None, Node), 'percentage_to_share': ('Percentage of the base node to share.', 0.50, float)}) class sd_crossover(SwapperDisruptor): @@ -746,7 +764,6 @@ def set_seed(self, prev_data): @disruptor(tactics, dtype="tCOMB", weight=1, - gen_args=GENERIC_ARGS, args={'node': ('Node to combine with.', None, Node)}) class sd_combine(SwapperDisruptor): """ @@ -896,6 +913,7 @@ def disrupt_data(self, dm, target, prev_data): prev_content = prev_data.content if isinstance(prev_content, Node): fuzz_data_tree(prev_content, self.path) + prev_data.altered = True else: prev_data.add_info('DONT_PROCESS_THIS_KIND_OF_DATA') @@ -1027,6 +1045,7 @@ def disrupt_data(self, dm, target, prev_data): prev_data.update_from(new_val) ret = prev_data + ret.altered = True return ret @@ -1074,15 +1093,18 @@ def disrupt_data(self, dm, target, prev_data): if self.new_val is None: if val != b'': val = corrupt_bits(val, n=1, ascii=self.ascii) - prev_data.add_info('corrupt data: {!s}'.format(truncate_info(val))) + prev_data.add_info('corrupted data: {!s}'.format(truncate_info(val))) else: prev_data.add_info('Nothing to corrupt!') else: val = self.new_val - prev_data.add_info('corrupt data: {!s}'.format(truncate_info(val))) + prev_data.add_info('corrupted data: {!s}'.format(truncate_info(val))) - i.set_values(values=[val]) - i.get_value() + status, _, _, _ = i.absorb(val, constraints=AbsNoCsts()) + if status != AbsorbStatus.FullyAbsorbed: + prev_data.add_info('data absorption failure, fallback to node replacement') + i.set_values(values=[val]) + i.freeze() ret = prev_data @@ -1092,6 +1114,7 @@ def disrupt_data(self, dm, target, prev_data): prev_data.add_info('Corruption performed on a byte string as no Node is available') ret = prev_data + ret.altered = True return ret @@ -1118,6 +1141,7 @@ def disrupt_data(self, dm, target, prev_data): msg = val[:self.idx-1]+new_value+val[self.idx:] prev_data.update_from(msg) + prev_data.altered = True return prev_data @@ -1126,8 +1150,8 @@ def disrupt_data(self, dm, target, prev_data): args={'path': ('Graph path regexp to select nodes on which ' \ 'the disruptor should apply.', None, str), 'clone_node': ('If True the dmaker will always return a copy ' \ - 'of the node. (For stateless diruptors dealing with ' \ - 'big data it can be usefull to it to False.)', False, bool)}) + 'of the node. (For stateless disruptors dealing with ' \ + 'big data it can be useful to it to False.)', False, bool)}) class d_fix_constraints(Disruptor): ''' Fix data constraints. @@ -1178,8 +1202,8 @@ def disrupt_data(self, dm, target, prev_data): 'the disruptor should apply.', None, str), 'recursive': ('Apply the disruptor recursively.', True, str), 'clone_node': ('If True the dmaker will always return a copy ' \ - 'of the node. (for stateless diruptors dealing with ' \ - 'big data it can be usefull to it to False).', False, bool)}) + 'of the node. (for stateless disruptors dealing with ' \ + 'big data it can be useful to it to False).', False, bool)}) class d_next_node_content(Disruptor): ''' Move to the next content of the nodes from input data or from only @@ -1223,6 +1247,75 @@ def disrupt_data(self, dm, target, prev_data): return prev_data +@disruptor(tactics, dtype="OP", weight=4, + args={'path': ('Graph path regexp to select nodes on which ' \ + 'the disruptor should apply.', None, str), + 'op': ('The operation to perform on the selected nodes.', Node.clear_attr, + (types.MethodType, types.FunctionType)), # python3, python2 + 'params': ('Tuple of parameters that will be provided to the operation. (' + 'default: MH.Attr.Mutable)', + (MH.Attr.Mutable,), + tuple), + 'clone_node': ('If True the dmaker will always return a copy ' \ + 'of the node. (For stateless disruptors dealing with ' \ + 'big data it can be useful to set it to False.)', False, bool)}) +class d_operate_on_nodes(Disruptor): + ''' + Perform an operation on the nodes specified by the regexp path. @op is an operation that + applies to a node and @params are a tuple containing the parameters that will be provided to + @op. If no path is provided, the root node will be used. + ''' + def setup(self, dm, user_input): + return True + + def disrupt_data(self, dm, target, prev_data): + ok = False + prev_content = prev_data.content + if not isinstance(prev_content, Node): + prev_data.add_info('INVALID INPUT') + return prev_data + + if self.path: + l = prev_content.get_reachable_nodes(path_regexp=self.path) + if not l: + prev_data.add_info('INVALID INPUT') + return prev_data + + for n in l: + try: + self.op(n, *self.params) + except: + prev_data.add_info("An error occurred while performing the operation on the " + "node '{:s}'".format(n.name)) + else: + ok = True + self._add_info(prev_data, n) + else: + try: + self.op(prev_content, *self.params) + except: + prev_data.add_info("An error occurred while performing the operation on the " + "node '{:s}'".format(prev_content.name)) + else: + ok = True + self._add_info(prev_data, prev_content) + + if ok: + prev_data.add_info("performed operation: {!r}".format(self.op)) + prev_data.add_info("parameters provided: {:s}" + .format(', '.join((str(x) for x in self.params)))) + + prev_content.freeze() + + if self.clone_node: + exported_node = Node(prev_content.name, base_node=prev_content, new_env=True) + prev_data.update_from(exported_node) + + prev_data.altered = True + return prev_data + + def _add_info(self, prev_data, n): + prev_data.add_info("changed node: {!s}".format(n.get_path_from(prev_data.content))) @disruptor(tactics, dtype="MOD", weight=4, args={'path': ('Graph path regexp to select nodes on which ' \ @@ -1230,8 +1323,8 @@ def disrupt_data(self, dm, target, prev_data): 'value': ('The new value to inject within the data.', '', str), 'constraints': ('Constraints for the absorption of the new value.', AbsNoCsts(), AbsCsts), 'clone_node': ('If True the dmaker will always return a copy ' \ - 'of the node. (For stateless diruptors dealing with ' \ - 'big data it can be usefull to it to False.)', False, bool)}) + 'of the node. (For stateless disruptors dealing with ' \ + 'big data it can be useful to it to False.)', False, bool)}) class d_modify_nodes(Disruptor): ''' Change the content of the nodes specified by the regexp path with @@ -1269,6 +1362,7 @@ def disrupt_data(self, dm, target, prev_data): exported_node = Node(prev_content.name, base_node=prev_content, new_env=True) prev_data.update_from(exported_node) + prev_data.altered = True return prev_data def _add_info(self, prev_data, n, status, size): @@ -1287,7 +1381,7 @@ def _add_info(self, prev_data, n, status, size): @disruptor(tactics, dtype="COPY", weight=4, - args={}) + args=None) class d_shallow_copy(Disruptor): ''' Shallow copy of the input data, which means: ignore its frozen diff --git a/framework/global_resources.py b/framework/global_resources.py index f4e5152..c148c09 100644 --- a/framework/global_resources.py +++ b/framework/global_resources.py @@ -31,7 +31,7 @@ from libs.utils import ensure_dir, ensure_file -fuddly_version = '0.26' +fuddly_version = '0.27' framework_folder = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # framework_folder = os.path.dirname(framework.__file__) @@ -59,6 +59,8 @@ ensure_dir(external_libs_folder) external_tools_folder = fuddly_data_folder + 'external_tools' + os.sep ensure_dir(external_tools_folder) +config_folder = os.path.join(fuddly_data_folder, 'config') + os.sep +ensure_dir(config_folder) user_projects_folder = fuddly_data_folder + 'user_projects' + os.sep ensure_dir(user_projects_folder) @@ -67,6 +69,10 @@ ensure_dir(user_data_models_folder) ensure_file(user_data_models_folder + os.sep + '__init__.py') +user_targets_folder = fuddly_data_folder + 'user_targets' + os.sep +ensure_dir(user_targets_folder) +ensure_file(user_targets_folder + os.sep + '__init__.py') + fmk_folder = app_folder + os.sep + 'framework' + os.sep internal_repr_codec = 'utf8' diff --git a/framework/knowledge/__init__.py b/framework/knowledge/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/framework/knowledge/feedback_collector.py b/framework/knowledge/feedback_collector.py new file mode 100644 index 0000000..2bd5eb0 --- /dev/null +++ b/framework/knowledge/feedback_collector.py @@ -0,0 +1,115 @@ +################################################################################ +# +# Copyright 2018 Eric Lacombe +# +################################################################################ +# +# This file is part of fuddly. +# +# fuddly is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# fuddly is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with fuddly. If not, see +# +################################################################################ + +import copy +import datetime +import threading +import collections + +class FeedbackSource(object): + + def __init__(self, src, subref=None, reliability=None, related_tg=None): + self._name = str(src) if subref is None else str(src) + ' - ' + str(subref) + self._obj = src + self._reliability = reliability + self._related_tg = related_tg + + def __str__(self): + return self._name + + def __hash__(self): + return id(self._obj) + + def __eq__(self, other): + return id(self._obj) == id(other._obj) + + @property + def obj(self): + return self._obj + + @property + def related_tg(self): + return self._related_tg + + +class FeedbackCollector(object): + fbk_lock = threading.Lock() + + def __init__(self): + self.cleanup() + self._feedback_collector = collections.OrderedDict() + self._feedback_collector_tstamped = collections.OrderedDict() + self._tstamped_bstring = None + + def add_fbk_from(self, ref, fbk, status=0): + now = datetime.datetime.now() + with self.fbk_lock: + if ref not in self._feedback_collector: + self._feedback_collector[ref] = {} + self._feedback_collector[ref]['data'] = [] + self._feedback_collector[ref]['status'] = 0 + self._feedback_collector_tstamped[ref] = [] + self._feedback_collector[ref]['data'].append(fbk) + self._feedback_collector[ref]['status'] = status + self._feedback_collector_tstamped[ref].append(now) + + def has_fbk_collector(self): + return len(self._feedback_collector) > 0 + + def __iter__(self): + with self.fbk_lock: + fbk_collector = copy.copy(self._feedback_collector) + fbk_collector_ts = copy.copy(self._feedback_collector_tstamped) + for ref, fbk in fbk_collector.items(): + yield ref, fbk['data'], fbk['status'], fbk_collector_ts[ref] + + def iter_and_cleanup_collector(self): + with self.fbk_lock: + fbk_collector = self._feedback_collector + fbk_collector_ts = self._feedback_collector_tstamped + self._feedback_collector = collections.OrderedDict() + self._feedback_collector_tstamped = collections.OrderedDict() + for ref, fbk in fbk_collector.items(): + yield ref, fbk['data'], fbk['status'], fbk_collector_ts[ref] + + def set_error_code(self, err_code): + self._err_code = err_code + + def get_error_code(self): + return self._err_code + + def set_bytes(self, bstring): + now = datetime.datetime.now() + self._tstamped_bstring = (bstring, now) + + def get_bytes(self): + return None if self._tstamped_bstring is None else self._tstamped_bstring[0] + + def get_timestamp(self): + return None if self._tstamped_bstring is None else self._tstamped_bstring[1] + + def cleanup(self): + # fbk_collector cleanup is done during consumption to avoid loss of feedback in + # multi-threading context + self._tstamped_bstring = None + self.set_error_code(0) \ No newline at end of file diff --git a/framework/knowledge/feedback_handler.py b/framework/knowledge/feedback_handler.py new file mode 100644 index 0000000..0ce905b --- /dev/null +++ b/framework/knowledge/feedback_handler.py @@ -0,0 +1,137 @@ +################################################################################ +# +# Copyright 2018 Eric Lacombe +# +################################################################################ +# +# This file is part of fuddly. +# +# fuddly is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# fuddly is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with fuddly. If not, see +# +################################################################################ + +import functools + +from framework.knowledge.information import * +import libs.debug_facility as dbg + +if dbg.KNOW_DEBUG: + DEBUG_PRINT = dbg.DEBUG_PRINT +else: + DEBUG_PRINT = dbg.NO_PRINT + +@functools.total_ordering +class SimilarityMeasure(object): + def __init__(self, level=0): + self._level = level + + @property + def value(self): + return self._level + + def __eq__(self, other): + return self._level == other._level + + def __lt__(self, other): + return self._level < other._level + + def __add__(self, other): + new_lvl = (self._level + other._level) // 2 + return SimilarityMeasure(level=new_lvl) + +UNIQUE = SimilarityMeasure(level=0) +EQUAL = SimilarityMeasure(level=16) +MID_SIMILAR = SimilarityMeasure(level=8) + + +class FeedbackHandler(object): + ''' + A feedback handler extract information from binary data. + ''' + + def notify_data_sending(self, data_list, timestamp, target): + ''' + *** To be overloaded *** + + This function is called when data have been sent. It enables to process feedback relatively + to previously sent data. + + Args: + data_list (list): list of :class:`framework.data.Data` that were sent + timestamp (datetime): date when data was sent + target (Target): target to which data was sent + ''' + pass + + def extract_info_from_feedback(self, source, timestamp, content, status): + ''' + *** To be overloaded *** + + Args: + source (:class:`framework.knowledge.feedback_collector.FeedbackSource`): source of the feedback + timestamp (datetime): date of reception of the feedback + content (bytes): binary data to process + status (int): negative status signify an error + + Returns: + Info: a set of :class:`.information.Info` or only one + ''' + return None + + def estimate_last_data_impact_uniqueness(self): + ''' + *** To be overloaded *** + + Estimate the similarity of the consequences triggered by the current data sending + from previous sending. + Estimation can be computed with provided feedback. + + Returns: + SimilarityMeasure: provide an estimation of impact similarity + ''' + return UNIQUE + + def process_feedback(self, source, timestamp, content, status): + info_set = set() + truncated_content = None if content is None else content[:60] + + DEBUG_PRINT( + '\n*** Feedback Entry ***\n' + ' source: {!s}\n' + ' timestamp: {!s}\n' + ' content: {!r} ...\n' + ' status: {!s}'.format(source, timestamp, truncated_content, status)) + + info = self.extract_info_from_feedback(source, timestamp, content, status) + if info is not None: + if isinstance(info, list): + for i in info: + info_set.add(i) + else: + info_set.add(info) + + return info_set + + +class TestFbkHandler(FeedbackHandler): + + def extract_info_from_feedback(self, source, timestamp, content, status): + if content is None: + return None + elif b'Linux' in content: + # OS.Linux.increase_trust() + return OS.Linux + elif b'Windows' in content: + # OS.Windows.increase_trust() + return OS.Windows diff --git a/framework/knowledge/information.py b/framework/knowledge/information.py new file mode 100644 index 0000000..afe17e8 --- /dev/null +++ b/framework/knowledge/information.py @@ -0,0 +1,162 @@ +################################################################################ +# +# Copyright 2018 Eric Lacombe +# +################################################################################ +# +# This file is part of fuddly. +# +# fuddly is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# fuddly is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with fuddly. If not, see +# +################################################################################ + +from enum import Enum + +try: + from enum import auto +except ImportError: + __my_enum_auto_id = 1 + def auto(): + global __my_enum_auto_id + i = __my_enum_auto_id + __my_enum_auto_id += 1 + return i + + +class TrustLevel(Enum): + Maximum = auto() + Medium = auto() + Minimum = auto() + +class Info(Enum): + def __init__(self, val): + self._trust = {} + + def increase_trust(self, inc=1): + if self.value not in self._trust: + self._trust[self.value] = 0 + self._trust[self.value] += inc + + def decrease_trust(self, inc=1): + if self.value not in self._trust: + self._trust[self.value] = 0 + self._trust[self.value] -= inc + + def reset_trust(self): + self._trust[self.value] = 0 + + def __str__(self): + name = self.__class__.__name__ + '.' + self.name + return 'Info: {!s} [{!s} --> value: {:d}]'.format( + name, self.trust_level, self.trust_value) + + @property + def trust_value(self): + return self._trust.get(self.value, 0) + + @property + def trust_level(self): + trust_values = self._trust.values() + if not trust_values: + return None + trust_val = self.trust_value + if trust_val >= max(trust_values): + return TrustLevel.Maximum + elif trust_val <= min(trust_values): + return TrustLevel.Minimum + else: + return TrustLevel.Medium + + +class OS(Info): + Linux = auto() + Windows = auto() + Android = auto() + Unknown = auto() + +class Hardware(Info): + X86_64 = auto() + X86_32 = auto() + PowerPc = auto() + ARM = auto() + Unknown = auto() + +class Language(Info): + C = auto() + Pascal = auto() + Unknown = auto() + +class InputHandling(Info): + Ctrl_Char_Set = auto() + Unknown = auto() + + +class InformationCollector(object): + + def __init__(self): + self._collector = None + self.reset_information() + + def add_information(self, info, initial_trust_value=0): + assert info is not None + + try: + for i in info: + assert isinstance(i, Info) + if i in self._collector: + i.increase_trust() + else: + i.increase_trust(inc=initial_trust_value) + self._collector.add(i) + except TypeError: + raise + # self._collector.add(info) + + def is_assumption_valid(self, info): + return not self._collector or info in self._collector + + def is_info_class_represented(self, info_class): + for info in self._collector: + if isinstance(info, info_class): + return True + else: + return False + + def reset_information(self): + self._collector = set() + + def __str__(self): + desc = '' + for info in self._collector: + desc += str(info) + '\n' + + return desc + + # for python2 compatibility + def __nonzero__(self): + return bool(self._collector) + + # for python3 compatibility + def __bool__(self): + return bool(self._collector) + + +if __name__ == "__main__": + + OS.Linux.increase_trust() + OS.Linux.increase_trust() + OS.Linux.show_trust() + + OS.Windows.show_trust() + diff --git a/framework/logger.py b/framework/logger.py index 3e189dd..ec3590e 100644 --- a/framework/logger.py +++ b/framework/logger.py @@ -28,9 +28,11 @@ import itertools from libs.external_modules import * +from libs.utils import get_caller_object from framework.data import Data from framework.global_resources import * from framework.database import Database +from framework.knowledge.feedback_collector import FeedbackSource from libs.utils import ensure_dir import framework.global_resources as gr @@ -43,14 +45,14 @@ class Logger(object): fmkDB = None - def __init__(self, name=None, prefix='', export_data=False, explicit_data_recording=False, + def __init__(self, name=None, prefix='', record_data=False, explicit_data_recording=False, export_orig=True, export_raw_data=True, console_display_limit=800, enable_file_logging=False): ''' Args: name (str): Name to be used in the log filenames. If not specified, the name of the project in which the logger is embedded will be used. - export_data (bool): If True, each emitted data will be stored in a specific + record_data (bool): If True, each emitted data will be stored in a specific file within `exported_data/`. explicit_data_recording (bool): Used for logging outcomes further to an Operator instruction. If True, the operator would have to state explicitly if it wants the just emitted data to be recorded. @@ -67,7 +69,7 @@ def __init__(self, name=None, prefix='', export_data=False, explicit_data_record ''' self.name = name self.p = prefix - self.__export_data = export_data + self.__record_data = record_data self.__explicit_data_recording = explicit_data_recording self.__export_orig = export_orig self._console_display_limit = console_display_limit @@ -75,7 +77,7 @@ def __init__(self, name=None, prefix='', export_data=False, explicit_data_record now = datetime.datetime.now() self.__prev_export_date = now.strftime("%Y%m%d_%H%M%S") self.__export_cpt = 0 - self.__export_raw_data = export_raw_data + self.export_raw_data = export_raw_data self._enable_file_logging = enable_file_logging self._fd = None @@ -86,13 +88,13 @@ def __init__(self, name=None, prefix='', export_data=False, explicit_data_record def init_logfn(x, nl_before=True, nl_after=False, rgb=None, style=None, verbose=False, do_record=True): if issubclass(x.__class__, Data): - data = repr(x) if self.__export_raw_data else str(x) + data = self._handle_binary_content(x.to_bytes(), raw=self.export_raw_data) rgb = None style = None - elif issubclass(x.__class__, bytes) and sys.version_info[0] > 2: - data = repr(x) if self.__export_raw_data else x.decode(internal_repr_codec) - else: + elif isinstance(x, str): data = x + else: + data = self._handle_binary_content(x, raw=self.export_raw_data) self.print_console(data, nl_before=nl_before, nl_after=nl_after, rgb=rgb, style=style) if verbose and issubclass(x.__class__, Data): x.show() @@ -101,13 +103,27 @@ def init_logfn(x, nl_before=True, nl_after=False, rgb=None, style=None, verbose= self.log_fn = init_logfn + def __str__(self): + return 'Logger' + + + def _handle_binary_content(self, content, raw=False): + content = gr.unconvert_from_internal_repr(content) + if sys.version_info[0] > 2: + content = content if not raw else '{!a}'.format(content) + else: + content = content if not raw else repr(content) + + return content + def start(self): self.__idx = 0 self.__tmp = False - self._reset_current_state() - self.last_data_id = None + self.reset_current_state() + self._current_sent_date = None + self._last_data_IDs = {} # per target_ref self.last_data_recordable = None with self._tg_fbk_lck: @@ -126,13 +142,13 @@ def start(self): def intern_func(x, nl_before=True, nl_after=False, rgb=None, style=None, verbose=False, do_record=True): if issubclass(x.__class__, Data): - data = repr(x) if self.__export_raw_data else str(x) + data = self._handle_binary_content(x.to_bytes(), raw=self.export_raw_data) rgb = None style = None - elif issubclass(x.__class__, bytes) and sys.version_info[0] > 2: - data = repr(x) if self.__export_raw_data else x.decode(internal_repr_codec) - else: + elif isinstance(x, str): data = x + else: + data = self._handle_binary_content(x, raw=self.export_raw_data) self.print_console(data, nl_before=nl_before, nl_after=nl_after, rgb=rgb, style=style) if not do_record: return data @@ -162,72 +178,75 @@ def stop(self): if self._fd: self._fd.close() - self._reset_current_state() - self.last_data_id = None + self.reset_current_state() + self._current_sent_date = None + self._last_data_IDs = {} self.last_data_recordable = None self.print_console('*** Logger is stopped ***\n', nl_before=False, rgb=Color.COMPONENT_STOP) - def _reset_current_state(self): + def reset_current_state(self): self._current_data = None + self._current_group_id = None self._current_orig_data_id = None self._current_size = None - self._current_sent_date = None - self._current_ack_date = None + self._current_ack_dates = None self._current_dmaker_list= [] self._current_dmaker_info = {} self._current_src_data_id = None self._current_fmk_info = [] - def commit_log_entry(self, group_id, prj_name, tg_name): + def commit_data_table_entry(self, group_id, prj_name): if self._current_data is not None: # that means data will be recorded init_dmaker = self._current_data.get_initial_dmaker() init_dmaker = Database.DEFAULT_GTYPE_NAME if init_dmaker is None else init_dmaker[0] dm = self._current_data.get_data_model() dm_name = Database.DEFAULT_DM_NAME if dm is None else dm.name + self._current_group_id = group_id + + last_data_id = None + for tg_ref, ack_date in self._current_ack_dates.items(): + last_data_id = self.fmkDB.insert_data(init_dmaker, dm_name, + self._current_data.to_bytes(), + self._current_size, + self._current_sent_date, + ack_date, + tg_ref, prj_name, + group_id=group_id) + # assert isinstance(tg_ref, FeedbackSource) + self._last_data_IDs[tg_ref.obj] = last_data_id + + if last_data_id is None: + print("\n*** ERROR: Cannot insert the data record in FMKDB!") + self.last_data_recordable = None + return last_data_id + + self._current_data.set_data_id(last_data_id) + + if self._current_orig_data_id is not None: + self.fmkDB.insert_steps(last_data_id, 1, None, None, + self._current_orig_data_id, + None, None) + step_id_start = 2 + else: + step_id_start = 1 - self.last_data_id = self.fmkDB.insert_data(init_dmaker, dm_name, - self._current_data.to_bytes(), - self._current_size, - self._current_sent_date, - self._current_ack_date, - tg_name, prj_name, - group_id=group_id) - - if self.last_data_id is None: - print("\n*** ERROR: Cannot insert the data record in FMKDB!") - self.last_data_id = None - self.last_data_recordable = None - self._reset_current_state() - return self.last_data_id - - self._current_data.set_data_id(self.last_data_id) - - if self._current_orig_data_id is not None: - self.fmkDB.insert_steps(self.last_data_id, 1, None, None, - self._current_orig_data_id, - None, None) - step_id_start = 2 - else: - step_id_start = 1 - - for step_id, dmaker in enumerate(self._current_dmaker_list, start=step_id_start): - dmaker_type, dmaker_name, user_input = dmaker - info = self._current_dmaker_info.get((dmaker_type,dmaker_name), None) - if info is not None: - info = '\n'.join(info) - info = convert_to_internal_repr(info) - self.fmkDB.insert_steps(self.last_data_id, step_id, dmaker_type, dmaker_name, - self._current_src_data_id, - str(user_input), info) - - for msg, now in self._current_fmk_info: - self.fmkDB.insert_fmk_info(self.last_data_id, msg, now) + for step_id, dmaker in enumerate(self._current_dmaker_list, start=step_id_start): + dmaker_type, dmaker_name, user_input = dmaker + info = self._current_dmaker_info.get((dmaker_type,dmaker_name), None) + if info is not None: + info = '\n'.join(info) + info = convert_to_internal_repr(info) + ui = str(user_input) if bool(user_input) else None + self.fmkDB.insert_steps(last_data_id, step_id, dmaker_type, dmaker_name, + self._current_src_data_id, + ui, info) - self._reset_current_state() + for msg, now in self._current_fmk_info: + self.fmkDB.insert_fmk_info(last_data_id, msg, now) - return self.last_data_id + return last_data_id else: return None @@ -242,14 +261,18 @@ def log_fmk_info(self, info, nl_before=False, nl_after=False, rgb=Color.FMKINFO, msg = "{prefix:s}*** [ {message:s} ] ***{suffix:s}".format(prefix=p, suffix=s, message=info) self.log_fn(msg, rgb=rgb) - data_id = self.last_data_id if data_id is None else data_id + if do_record: if not delay_recording: - self.fmkDB.insert_fmk_info(data_id, info, now) + if data_id is None: + for d_id in self._last_data_IDs.values(): + self.fmkDB.insert_fmk_info(d_id, info, now) + else: + self.fmkDB.insert_fmk_info(data_id, info, now) else: self._current_fmk_info.append((info, now)) - def collect_target_feedback(self, fbk, status_code=None): + def collect_feedback(self, content, status_code=None): """ Used within the scope of the Logger feedback-collector infrastructure. If your target implement the interface :meth:`Target.get_feedback`, no need to @@ -258,22 +281,71 @@ def collect_target_feedback(self, fbk, status_code=None): To be called by the target each time feedback need to be registered. Args: - fbk: feedback record + content: feedback record status_code (int): should be negative for error """ now = datetime.datetime.now() + fbk_src = get_caller_object() with self._tg_fbk_lck: - self._tg_fbk.append((now, fbk, status_code)) + self._tg_fbk.append((now, FeedbackSource(fbk_src), content, status_code)) + + def shall_record(self): + if self.last_data_recordable or not self.__explicit_data_recording: + return True + else: + # feedback will not be recorded because data is not recorded + return False + + def _log_feedback(self, source, content, status_code, timestamp, record=True): + + processed_feedback = self._process_target_feedback(content) + fbk_cond = status_code is not None and status_code < 0 + hdr_color = Color.FEEDBACK_ERR if fbk_cond else Color.FEEDBACK + body_color = Color.FEEDBACK_HLIGHT if fbk_cond else None + if not processed_feedback: + msg_hdr = "### Status from '{!s}': {!s}".format(source, status_code) + else: + msg_hdr = "### Feedback from '{!s}' (status={!s}):".format(source, status_code) + self.log_fn(msg_hdr, rgb=hdr_color, do_record=record) + if processed_feedback: + if isinstance(processed_feedback, list): + for dfbk in processed_feedback: + self.log_fn(dfbk, rgb=body_color, do_record=record) + else: + self.log_fn(processed_feedback, rgb=body_color, do_record=record) + + if record: + assert isinstance(source, FeedbackSource) + if source.related_tg is not None: + try: + data_id = self._last_data_IDs[source.related_tg] + except KeyError: + print('\nWarning: The feedback source is related to a target to which nothing has been sent.' + ' Retrieved feedback will not be attached to any data ID.') + data_id = None + else: + ids = self._last_data_IDs.values() + data_id = max(ids) if ids else None + + if isinstance(content, list): + for fbk, ts in zip(content, timestamp): + self.fmkDB.insert_feedback(data_id, source, ts, + self._encode_target_feedback(fbk), + status_code=status_code) + else: + self.fmkDB.insert_feedback(data_id, source, timestamp, + self._encode_target_feedback(content), + status_code=status_code) - def log_collected_target_feedback(self, preamble=None, epilogue=None): + def log_collected_feedback(self, preamble=None, epilogue=None): """ Used within the scope of the Logger feedback-collector feature. If your target implement the interface :meth:`Target.get_feedback`, no need to use this infrastructure. It allows to retrieve the collected feedback, that has been populated - by the target (through call to :meth:`Logger.collect_target_feedback`). + by the target (through call to :meth:`Logger.collect_feedback`). Args: preamble (str): prefix added to each collected feedback @@ -281,9 +353,9 @@ def log_collected_target_feedback(self, preamble=None, epilogue=None): Returns: bool: True if target feedback has been collected through logger infrastructure - :meth:`Logger.collect_target_feedback`, False otherwise. + :meth:`Logger.collect_feedback`, False otherwise. """ - error_detected = False + error_detected = {} with self._tg_fbk_lck: fbk_list = self._tg_fbk @@ -293,126 +365,49 @@ def log_collected_target_feedback(self, preamble=None, epilogue=None): # self.log_fn("\n::[ NO TARGET FEEDBACK ]::\n") raise NotImplementedError - if self.last_data_recordable or not self.__explicit_data_recording: - record = True - else: - # feedback will not be recorded because data is not recorded - record = False + record = self.shall_record() if preamble is not None: self.log_fn(preamble, do_record=record, rgb=Color.FMKINFO) - for fbk, idx in zip(fbk_list, range(len(fbk_list))): - timestamp, m, status = fbk - fbk_cond = status is not None and status < 0 - hdr_color = Color.FEEDBACK_ERR if fbk_cond else Color.FEEDBACK - body_color = Color.FEEDBACK_HLIGHT if fbk_cond else None - self.log_fn("### Collected Target Feedback [{:d}] (status={!s}): ".format(idx, status), - rgb=hdr_color, do_record=record) - self.log_fn(m, rgb=body_color, do_record=record) - if record: - self.fmkDB.insert_feedback(self.last_data_id, - "Collector [record #{:d}]".format(idx), - timestamp, - self._encode_target_feedback(m), - status_code=status) + for idx, fbk_record in enumerate(fbk_list): + timestamp, fbk_src, fbk, status = fbk_record + self._log_feedback(fbk_src, fbk, status, timestamp, record=record) + if status is not None and status < 0: - error_detected = True + error_detected[fbk_src.obj] = True + else: + error_detected[fbk_src.obj] = False if epilogue is not None: self.log_fn(epilogue, do_record=record, rgb=Color.FMKINFO) return error_detected - def log_target_feedback_from(self, feedback, timestamp, - preamble=None, epilogue=None, - source=None, - status_code=None): - decoded_feedback = self._decode_target_feedback(feedback) - if self.last_data_recordable or not self.__explicit_data_recording: - record = True - else: - # feedback will not be recorded because data is not recorded - record = False + def log_target_feedback_from(self, source, content, status_code, timestamp, + preamble=None, epilogue=None): + record = self.shall_record() if preamble is not None: self.log_fn(preamble, do_record=record, rgb=Color.FMKINFO) - if not decoded_feedback and (status_code is None or status_code >= 0): - msg_hdr = "### No Target Feedback!" if source is None else '### No Target Feedback from "{!s}"!'.format( - source) - self.log_fn(msg_hdr, rgb=Color.FEEDBACK, do_record=record) - else: - fbk_cond = status_code is not None and status_code < 0 - hdr_color = Color.FEEDBACK_ERR if fbk_cond else Color.FEEDBACK - body_color = Color.FEEDBACK_HLIGHT if fbk_cond else None - if not decoded_feedback: - msg_hdr = "### Target Status: {!s}".format(status_code) if source is None \ - else "### Target Status from '{!s}': {!s}".format(source, status_code) - else: - msg_hdr = "### Target Feedback (status={!s}):".format(status_code) if source is None \ - else "### Target Feedback from '{!s}' (status={!s}):".format(source, status_code) - self.log_fn(msg_hdr, rgb=hdr_color, do_record=record) - if decoded_feedback: - if isinstance(decoded_feedback, list): - for dfbk in decoded_feedback: - self.log_fn(dfbk, rgb=body_color, do_record=record) - else: - self.log_fn(decoded_feedback, rgb=body_color, do_record=record) - - if record: - src = 'Default' if source is None else source - if isinstance(feedback, list): - for fbk, ts in zip(feedback, timestamp): - self.fmkDB.insert_feedback(self.last_data_id, src, ts, - self._encode_target_feedback(fbk), - status_code=status_code) - else: - self.fmkDB.insert_feedback(self.last_data_id, src, timestamp, - self._encode_target_feedback(feedback), - status_code=status_code) + self._log_feedback(source, content, status_code, timestamp, record=record) if epilogue is not None: self.log_fn(epilogue, do_record=record, rgb=Color.FMKINFO) - def log_operator_feedback(self, feedback, timestamp, op_name, status_code=None): - if feedback is None: - decoded_feedback = None - else: - decoded_feedback = self._decode_target_feedback(feedback) - # decoded_feedback can be the empty string - if self.last_data_recordable or not self.__explicit_data_recording: - record = True - else: - # feedback will not be recorded because data is not recorded - record = False + def log_operator_feedback(self, operator, content, status_code, timestamp): + self._log_feedback(FeedbackSource(operator), content, status_code, timestamp, + record=self.shall_record()) - if not decoded_feedback and status_code is None: - self.log_fn("### No Operator Feedback!", rgb=Color.FEEDBACK, - do_record=record) - else: - fbk_cond = status_code is not None and status_code < 0 - hdr_color = Color.FEEDBACK_ERR if fbk_cond else Color.FEEDBACK - body_color = Color.FEEDBACK_HLIGHT if fbk_cond else None - if decoded_feedback: - self.log_fn("### Operator Feedback (status={!s}):".format(status_code), - rgb=hdr_color, do_record=record) - self.log_fn(decoded_feedback, rgb=body_color, do_record=record) - else: # status_code is not None - self.log_fn("### Operator Status: {:d}".format(status_code), - rgb=hdr_color, do_record=record) - - if self.last_data_id is not None and record: - feedback = None if feedback is None else self._encode_target_feedback(feedback) - self.fmkDB.insert_feedback(self.last_data_id, - "Operator '{:s}'".format(op_name), - timestamp, - feedback, - status_code=status_code) + def log_probe_feedback(self, probe, content, status_code, timestamp, related_tg=None): + self._log_feedback(FeedbackSource(probe, related_tg=related_tg), content, status_code, timestamp, + record=self.shall_record()) - def _decode_target_feedback(self, feedback): + + def _process_target_feedback(self, feedback): if feedback is None: return feedback @@ -420,17 +415,15 @@ def _decode_target_feedback(self, feedback): new_fbk = [] for f in feedback: new_f = f.strip() - if sys.version_info[0] > 2 and new_f and isinstance(new_f, bytes): - new_f = new_f.decode(internal_repr_codec, 'replace') - new_f = eval('{!a}'.format(new_f)) + if isinstance(new_f, bytes): + new_f = self._handle_binary_content(new_f, raw=self.export_raw_data) new_fbk.append(new_f) if not list(filter(lambda x: x != b'', new_fbk)): new_fbk = None else: new_fbk = feedback.strip() - if sys.version_info[0] > 2 and new_fbk and isinstance(new_fbk, bytes): - new_fbk = new_fbk.decode(internal_repr_codec, 'replace') - new_fbk = eval('{!a}'.format(new_fbk)) + if isinstance(new_fbk, bytes): + new_fbk = self._handle_binary_content(new_fbk, raw=self.export_raw_data) return new_fbk @@ -439,29 +432,6 @@ def _encode_target_feedback(self, feedback): return None return convert_to_internal_repr(feedback) - def log_probe_feedback(self, source, timestamp, content, status_code, force_record=False): - if self.last_data_recordable or not self.__explicit_data_recording or force_record: - record = True - else: - # feedback will not be recorded because data is not recorded - record = False - - fbk_cond = status_code is not None and status_code < 0 - hdr_color = Color.FEEDBACK_ERR if fbk_cond else Color.FEEDBACK - body_color = Color.FEEDBACK_HLIGHT if fbk_cond else None - if content is None: - self.log_fn("### {:s} Status: {:d}".format(source, status_code), - rgb=hdr_color, do_record=record) - else: - self.log_fn("### {:s} Feedback (status={:d}):".format(source, status_code), - rgb=hdr_color, do_record=record) - self.log_fn(self._decode_target_feedback(content),rgb=body_color, - do_record=record) - - if record: - content = None if content is None else self._encode_target_feedback(content) - self.fmkDB.insert_feedback(self.last_data_id, source, timestamp, content, - status_code=status_code) def start_new_log_entry(self, preamble=''): self.__idx += 1 @@ -471,6 +441,8 @@ def start_new_log_entry(self, preamble=''): msg += '='*(max(80-len(msg),0)) self.log_fn(msg, rgb=Color.NEWLOGENTRY, style=FontStyle.BOLD) + return self._current_sent_date + def log_dmaker_step(self, num): msg = "### Step %d:" % num self.log_fn(msg, rgb=Color.DMAKERSTEP) @@ -516,12 +488,17 @@ def log_info(self, info): msg = "### Info: {:s}".format(info) self.log_fn(msg, rgb=Color.INFO) - def log_target_ack_date(self, date): - self._current_ack_date = date + def log_target_ack_date(self): + for tg_ref, ack_date in self._current_ack_dates.items(): + msg = "### Ack from '{!s}' received at: ".format(tg_ref) + self.log_fn(msg, nl_after=False, rgb=Color.LOGSECTION) + self.log_fn(str(ack_date), nl_before=False) - msg = "### Target ack received at: " - self.log_fn(msg, nl_after=False, rgb=Color.LOGSECTION) - self.log_fn(str(self._current_ack_date), nl_before=False) + def set_target_ack_date(self, tg_ref, date): + if self._current_ack_dates is None: + self._current_ack_dates = {tg_ref: date} + else: + self._current_ack_dates[tg_ref] = date def log_orig_data(self, data): @@ -533,7 +510,7 @@ def log_orig_data(self, data): if data is not None: self._current_orig_data_id = data.get_data_id() - if self.__export_orig and not self.__export_data: + if self.__export_orig and not self.__record_data: if data is None: msgs = ("### No Original Data",) else: @@ -578,7 +555,7 @@ def log_data(self, data, verbose=False): self._current_data = data self.last_data_recordable = self._current_data.is_recordable() - if not self.__export_data: + if not self.__record_data: self.log_fn("### Data emitted:", rgb=Color.LOGSECTION) self.log_fn(data, nl_after=True, verbose=verbose) else: @@ -636,14 +613,16 @@ def log_comment(self, comment): self.log_fn("### Comments [{date:s}]:".format(date=current_date), rgb=Color.COMMENTS) self.log_fn(comment) - self.fmkDB.insert_comment(self.last_data_id, comment, now) + for data_id in self._last_data_IDs.values(): + self.fmkDB.insert_comment(data_id, comment, now) self.print_console('\n') def log_error(self, err_msg): now = datetime.datetime.now() msg = "\n/!\\ ERROR: %s /!\\\n" % err_msg self.log_fn(msg, rgb=Color.ERROR) - self.fmkDB.insert_fmk_info(self.last_data_id, msg, now, error=True) + for data_id in self._last_data_IDs.values(): + self.fmkDB.insert_fmk_info(data_id, msg, now, error=True) def print_console(self, msg, nl_before=True, nl_after=False, rgb=None, style=None, raw_limit=None, limit_output=True): @@ -656,7 +635,7 @@ def print_console(self, msg, nl_before=True, nl_after=False, rgb=None, style=Non prefix = p + self.p - if (sys.version_info[0] > 2 and isinstance(msg, bytes)) or issubclass(msg.__class__, Data): + if isinstance(msg, Data): msg = repr(msg) suffix = '' diff --git a/framework/monitor.py b/framework/monitor.py index 1f06ff9..1b900ae 100644 --- a/framework/monitor.py +++ b/framework/monitor.py @@ -36,8 +36,8 @@ class ProbeUser(object): - timeout = 10.0 - probe_init_timeout = 20.0 + timeout = 5.0 + probe_init_timeout = 10.0 def __init__(self, probe): self._probe = probe @@ -45,6 +45,10 @@ def __init__(self, probe): self._started_event = threading.Event() self._stop_event = threading.Event() + @property + def probe(self): + return self._probe + def start(self, *args, **kwargs): if self.is_alive(): raise RuntimeError @@ -62,7 +66,7 @@ def join(self, timeout=None): self._thread.join(ProbeUser.timeout if timeout is None else timeout) if self.is_alive(): - raise ProbeTimeoutError(self.__class__.__name__, timeout, ["start()", "arm()", "main()", "stop()"]) + raise ProbeTimeoutError(self._probe.__class__.__name__, timeout, ["start()", "arm()", "main()", "stop()"]) self._stop_event.clear() @@ -119,7 +123,7 @@ def _wait_for_probe(self, event, timeout=None): while not event.is_set(): if (datetime.datetime.now() - start).total_seconds() >= timeout: self.stop() - raise ProbeTimeoutError(self.__class__.__name__, timeout) + raise ProbeTimeoutError(self._probe.__class__.__name__, timeout) if not self.is_alive() or not self._go_on(): break event.wait(1) @@ -311,11 +315,11 @@ class Monitor(object): def __init__(self): self.fmk_ops = None self._logger = None - self._target = None + self._targets = None self._target_status = None self._dm = None - self.probe_users = {} + self._tg_from_probe = {} self.__enable = True @@ -325,8 +329,8 @@ def set_fmk_ops(self, fmk_ops): def set_logger(self, logger): self._logger = logger - def set_target(self, target): - self._target = target + def set_targets(self, targets): + self._targets = targets def set_data_model(self, dm): self._dm = dm @@ -355,11 +359,15 @@ def disable_hooks(self): self.__enable = False def _get_probe_ref(self, probe): + # TODO: provide unique ref for same probe class name if isinstance(probe, type) and issubclass(probe, Probe): return probe.__name__ + elif isinstance(probe, Probe): + return probe.__class__.__name__ elif isinstance(probe, str): return probe else: + print(probe) raise TypeError def configure_probe(self, probe, *args): @@ -369,19 +377,28 @@ def configure_probe(self, probe, *args): return False return True - def start_probe(self, probe): - probe_name = self._get_probe_ref(probe) - if probe_name in self.probe_users: + def start_probe(self, probe, related_tg=None): + probe_ref = self._get_probe_ref(probe) + if probe_ref in self.probe_users: try: - self.probe_users[probe_name].start(self._dm, self._target, self._logger) + if related_tg is None: + self.probe_users[probe_ref].start(self._dm, self._targets, self._logger) + else: + self.probe_users[probe_ref].start(self._dm, related_tg, self._logger) except: + self.fmk_ops.set_error("Exception raised in probe '{:s}' start".format(probe_ref), + code=Error.UserCodeError) return False + else: + self._tg_from_probe[probe_ref] = related_tg return True def stop_probe(self, probe): probe_name = self._get_probe_ref(probe) if probe_name in self.probe_users: self.probe_users[probe_name].stop() + if probe_name in self._tg_from_probe: + del self._tg_from_probe[probe_name] self._wait_for_specific_probes(ProbeUser, ProbeUser.join, [probe]) else: self.fmk_ops.set_error("Probe '{:s}' does not exist".format(probe_name), @@ -390,10 +407,13 @@ def stop_probe(self, probe): def stop_all_probes(self): for _, probe_user in self.probe_users.items(): probe_user.stop() - + self._tg_from_probe = {} self._wait_for_specific_probes(ProbeUser, ProbeUser.join) + def get_probe_related_tg(self, probe): + return self._tg_from_probe[self._get_probe_ref(probe)] + def get_probe_status(self, probe): return self.probe_users[self._get_probe_ref(probe)].get_probe_status() @@ -415,6 +435,10 @@ def get_probes_names(self): probes_names.append(probe_name) return probes_names + def iter_probes(self): + for _, probeuser in self.probe_users.items(): + yield probeuser.probe + def _wait_for_specific_probes(self, probe_user_class, probe_user_wait_method, probes=None, timeout=None): """ @@ -499,7 +523,7 @@ def target_status(self): for n, probe_user in self.probe_users.items(): if probe_user.is_alive(): probe_status = probe_user.get_probe_status() - if probe_status.get_status() < 0: + if probe_status.value < 0: self._target_status = -1 break else: @@ -518,6 +542,9 @@ def __init__(self, delay=1.0): self._status = ProbeStatus(0) self._delay = delay + def __str__(self): + return "Probe - {:s}".format(self.__class__.__name__) + @property def status(self): return self._status @@ -616,24 +643,26 @@ class ProbeStatus(object): def __init__(self, status=None, info=None): self._now = None - self.__status = status - self.__private = info + self._status = status + self._private = info - def set_status(self, status): + @property + def value(self): + return self._status + + @value.setter + def value(self, val): """ Args: - status (int): negative status if something is wrong + val (int): negative value if something is wrong """ - self.__status = status - - def get_status(self): - return self.__status + self._status = val def set_private_info(self, pv): - self.__private = pv + self._private = pv def get_private_info(self): - return self.__private + return self._private def set_timestamp(self): self._now = datetime.datetime.now() @@ -998,18 +1027,18 @@ def main(self, dm, target, logger): status = ProbeStatus() if current_pid == -10: - status.set_status(-10) + status.value = -10 status.set_private_info("ERROR with the command") elif current_pid == -1: - status.set_status(-2) + status.value = -2 status.set_private_info("'{:s}' is not running anymore!".format(self.process_name)) elif self._saved_pid != current_pid: self._saved_pid = current_pid - status.set_status(-1) + status.value = -1 status.set_private_info("'{:s}' PID({:d}) has changed!".format(self.process_name, current_pid)) else: - status.set_status(current_pid) + status.value = current_pid status.set_private_info(None) return status @@ -1090,10 +1119,10 @@ def main(self, dm, target, logger): status = ProbeStatus() if current_mem == -10: - status.set_status(-10) + status.value = -10 status.set_private_info("ERROR with the command") elif current_mem == -1: - status.set_status(-2) + status.value = -2 status.set_private_info("'{:s}' is not found!".format(self.process_name)) else: if current_mem > self._max_mem: @@ -1112,11 +1141,11 @@ def main(self, dm, target, logger): ok = False err_msg += '\n*** Tolerance exceeded (original RSS: {:d}) ***'.format(self._saved_mem) if not ok: - status.set_status(-1) + status.value = -1 status.set_private_info(err_msg+'\n'+info) self._last_status_ok = False else: - status.set_status(self._max_mem) + status.value = self._max_mem status.set_private_info(info) self._last_status_ok = True return status diff --git a/framework/node.py b/framework/node.py index 1fc26a1..27da975 100644 --- a/framework/node.py +++ b/framework/node.py @@ -34,8 +34,11 @@ import collections import traceback import uuid +import struct +import math from enum import Enum +from random import shuffle sys.path.append('.') @@ -150,10 +153,10 @@ def make_private(self, node_dico): elif isinstance(node_containers, (tuple, list)): new_node_containers = [] for ctr in node_containers: - if isinstance(node_containers, Node): + if isinstance(ctr, Node): node, param = ctr, None else: - assert isinstance(node_containers, (tuple, list)) and len(node_containers) == 2 + assert isinstance(ctr, (tuple, list)) and len(ctr) == 2 node, param = ctr new_node = node_dico.get(node, None) if new_node is not None: @@ -217,7 +220,7 @@ def set_size_on_source_node(self, size): if not ok: print("\n*** WARNING: The node '{:s}' is not compatible with the integer" " '{:d}'".format(self._node.name, size)) - self._node.set_frozen_value(self._node.get_current_encoded_value()) + self._node.set_frozen_value(self._node.get_current_value()) def _sync_nodes_specific(self, src_node): if self.apply_to_enc_size: @@ -581,10 +584,12 @@ class NodeInternals(object): default_custo = None def __init__(self, arg=None): + # if new attributes are added, set_contents_from() have to be updated self.private = None self.absorb_helper = None self.absorb_constraints = None self.custo = None + self._env = None self.__attrs = { ### GENERIC ### @@ -610,6 +615,32 @@ def __init__(self, arg=None): self.customize(self.default_custo) self._init_specific(arg) + def set_contents_from(self, node_internals): + if node_internals is None or node_internals.__class__ == NodeInternals_Empty: + return + + self._env = node_internals._env + self.private = node_internals.private + self.__attrs = node_internals.__attrs + self._sync_with = node_internals._sync_with + self.absorb_constraints = node_internals.absorb_constraints + + if self.__class__ == node_internals.__class__: + self.custo = node_internals.custo + self.absorb_helper = node_internals.absorb_helper + else: + if self._sync_with is not None and SyncScope.Size in self._sync_with: + # This SyncScope is currently only supported by String-based + # NodeInternals_TypedValue + del self._sync_with[SyncScope.Size] + + def get_attrs_copy(self): + return (copy.copy(self.__attrs), copy.copy(self.custo)) + + def set_attrs_from(self, all_attrs): + self.__attrs = all_attrs[0] + self.custo = all_attrs[1] + def _init_specific(self, arg): pass @@ -622,6 +653,14 @@ def get_raw_value(self, **kwargs): def customize(self, custo): self.custo = copy.copy(custo) + @property + def env(self): + return self._env + + @env.setter + def env(self, src): + self._env = src + def has_subkinds(self): return False @@ -652,7 +691,6 @@ def synchronize_nodes(self, src_node): if isinstance(obj, SyncObj): obj.synchronize_nodes(src_node) - def make_private(self, ignore_frozen_state, accept_external_entanglement, delayed_node_internals, forget_original_sync_objs=False): if self.private is not None: @@ -670,31 +708,6 @@ def make_private(self, ignore_frozen_state, accept_external_entanglement, delaye self._make_private_specific(ignore_frozen_state, accept_external_entanglement) self.custo = copy.copy(self.custo) - def set_contents_from(self, node_internals): - if node_internals is None or node_internals.__class__ == NodeInternals_Empty: - return - - self.private = node_internals.private - self.__attrs = node_internals.__attrs - self._sync_with = node_internals._sync_with - self.absorb_constraints = node_internals.absorb_constraints - - if self.__class__ == node_internals.__class__: - self.custo = node_internals.custo - self.absorb_helper = node_internals.absorb_helper - else: - if self._sync_with is not None and SyncScope.Size in self._sync_with: - # This SyncScope is currently only supported by String-based - # NodeInternals_TypedValue - del self._sync_with[SyncScope.Size] - - def get_attrs_copy(self): - return (copy.copy(self.__attrs), copy.copy(self.custo)) - - def set_attrs_from(self, all_attrs): - self.__attrs = all_attrs[0] - self.custo = all_attrs[1] - # Called near the end of Node copy (Node.set_contents) to update # node references inside the NodeInternals def _update_node_refs(self, node_dico, debug): @@ -1182,7 +1195,8 @@ def get_raw_value(self, **kwargs): return Node.DEFAULT_DISABLED_VALUE def set_child_env(self, env): - print('Empty:', hex(id(self))) + self.env = env + print('\n*** Empty Node: {!s}'.format(hex(id(self)))) raise AttributeError @@ -1195,7 +1209,6 @@ def _init_specific(self, arg): self.generator_func = None self.generator_arg = None self.node_arg = None - self.env = None self.pdepth = 0 self._node_helpers = DynNode_Helpers() self.provide_helpers = False @@ -1335,7 +1348,8 @@ def make_args_private(self, node_dico, entangled_set, ignore_frozen_state, accep def reset_generator(self): self._generated_node = None - def _get_generated_node(self): + @property + def generated_node(self): if self._generated_node is None: if self.generator_arg is not None and self.node_arg is not None: @@ -1375,7 +1389,7 @@ def _get_generated_node(self): return self._generated_node - generated_node = property(fget=_get_generated_node) + # generated_node = property(fget=_get_generated_node) def import_generator_func(self, generator_func, generator_node_arg=None, generator_arg=None, @@ -1484,9 +1498,8 @@ def is_frozen(self): def unfreeze(self, conf=None, recursive=True, dont_change_state=False, ignore_entanglement=False, only_generators=False, reevaluate_constraints=False): - # if self.is_attr_set(NodeInternals.Mutable): - # print(self.custo.reset_on_unfreeze_mode, self.generated_node.name, - # self.default_custo.reset_on_unfreeze_mode) + # if self.is_attr_set(NodeInternals.DEBUG): + # print('\n*** DBG Gen:', self.custo.reset_on_unfreeze_mode) if self.custo.reset_on_unfreeze_mode: # 'dont_change_state' is not supported in this case. But # if generator is stateless, it should not be a problem. @@ -1512,13 +1525,11 @@ def reset_fuzz_weight(self, recursive): self.generated_node.reset_fuzz_weight(recursive=recursive) def set_child_env(self, env): - # if self._generated_node is not None: - # self._generated_node.set_env(env) - # self.env = env - self.set_env(env) - - def set_env(self, env): self.env = env + + @NodeInternals.env.setter + def env(self, env): + NodeInternals.env.fset(self, env) if self._generated_node is not None: self._generated_node.set_env(env) @@ -1712,7 +1723,7 @@ def reset_fuzz_weight(self, recursive): pass def set_child_env(self, env): - pass + self.env = env def set_child_attr(self, name, conf=None, all_conf=False, recursive=False): pass @@ -1735,6 +1746,7 @@ def get_child_all_path(self, name, htable, conf, recursive): class NodeInternals_TypedValue(NodeInternals_Term): + def _init_specific(self, arg): NodeInternals_Term._init_specific(self, arg) self.value_type = None @@ -1752,11 +1764,19 @@ def _unmake_specific(self, name): def import_value_type(self, value_type): self.value_type = value_type + # if self.env is not None: + # self.value_type.knowledge_source = self.env.knowledge_source if self.is_attr_set(NodeInternals.Determinist): self.value_type.make_determinist() else: self.value_type.make_random() + # @NodeInternals.env.setter + # def env(self, env): + # NodeInternals.env.fset(self, env) + # if self.value_type is not None and self.env is not None: + # self.value_type.knowledge_source = self.env.knowledge_source + def has_subkinds(self): return True @@ -1771,6 +1791,7 @@ def set_size_from_constraints(self, size, encoded_size): def set_specific_fuzzy_values(self, vals): self.__fuzzy_values = vals + self.value_type.add_specific_fuzzy_vals(vals) def get_specific_fuzzy_values(self): return self.__fuzzy_values @@ -1839,7 +1860,6 @@ def _init_specific(self, arg): self.fct = None self.node_arg = None self.fct_arg = None - self.env = None self._node_helpers = DynNode_Helpers() self.provide_helpers = False @@ -1880,9 +1900,6 @@ def set_func_arg(self, node=None, fct_arg=None): if fct_arg is not None: self.fct_arg = fct_arg - def set_env(self, env): - self.env = env - def customize(self, custo): if custo is None: self.custo = copy.copy(self.default_custo) @@ -2407,6 +2424,7 @@ def make_private_subnodes(self, node_dico, func_nodes, env, ignore_frozen_state, e.entangled_nodes = None for c in e.internals: + e.internals[c].env = env if e.is_nonterm(c): e.internals[c].make_private_subnodes(node_dico, func_nodes, env, ignore_frozen_state=ignore_frozen_state, @@ -2418,7 +2436,6 @@ def make_private_subnodes(self, node_dico, func_nodes, env, ignore_frozen_state, delayed_node_internals=delayed_node_internals) elif e.is_func(c) or e.is_genfunc(c): - e.internals[c].set_env(env) if e.internals[c].node_arg is not None: func_nodes.add(e) e.internals[c].make_private(ignore_frozen_state=ignore_frozen_state, @@ -2430,7 +2447,6 @@ def make_private_subnodes(self, node_dico, func_nodes, env, ignore_frozen_state, accept_external_entanglement=accept_external_entanglement, delayed_node_internals=delayed_node_internals) - def get_subnodes_csts_copy(self, node_dico=None): node_dico = {} if node_dico is None else node_dico # node_dico[old_node] --> new_node csts_copy = [] @@ -2621,8 +2637,13 @@ def _get_next_random_component(comp_list, excluded_idx=[], seed=None): return ret - # To be used only in Finite mode def structure_will_change(self): + ''' + To be used only in Finite mode. + Return True if the structure will change the next time _get_value() will be called. + + Returns: bool + ''' crit1 = (len(self.subnodes_order) // 2) > 1 @@ -2695,7 +2716,7 @@ def _copy_nodelist(self, node_list): new_list.append([delim, [sublist[0], copy.copy(sublist[1])]]) return new_list - def _generate_expanded_nodelist(self, node_list): + def _generate_expanded_nodelist(self, node_list, determinist=True): expanded_node_list = [] for idx, delim, sublist in self.__iter_csts_verbose(node_list): @@ -2739,6 +2760,9 @@ def _generate_expanded_nodelist(self, node_list): if not expanded_node_list: expanded_node_list.append(node_list) + if not determinist: + shuffle(expanded_node_list) + return expanded_node_list def _construct_subnodes(self, node_desc, subnode_list, mode, ignore_sep_fstate, ignore_separator=False, lazy_mode=True): @@ -2922,7 +2946,7 @@ def get_subnodes_with_csts(self): seed=self.component_seed) # If the shape is Pick (=+), the shape is reduced to a singleton - self.expanded_nodelist = self._generate_expanded_nodelist(node_list) + self.expanded_nodelist = self._generate_expanded_nodelist(node_list, determinist=determinist) self.expanded_nodelist_origsz = len(self.expanded_nodelist) if self.expanded_nodelist_sz > 0: @@ -2930,7 +2954,7 @@ def get_subnodes_with_csts(self): else: self.expanded_nodelist = self.expanded_nodelist[:1] elif not self.expanded_nodelist: # that is == [] - self.expanded_nodelist = self._generate_expanded_nodelist(node_list) + self.expanded_nodelist = self._generate_expanded_nodelist(node_list, determinist=determinist) self.expanded_nodelist_origsz = len(self.expanded_nodelist) node_list = self.expanded_nodelist.pop(-1) @@ -2955,11 +2979,11 @@ def get_subnodes_with_csts(self): if sublist[0] > -1: node = self._get_heavier_component(sublist[1], check_existence=True) else: - for n in sublist[1]: - node, _, _ = self._get_node_and_attrs_from(n) + for ndesc in sublist[1]: + node, _, _ = self._get_node_and_attrs_from(ndesc) shall_exist = self._existence_from_node(node) if shall_exist is None or shall_exist: - node = n + node = ndesc break else: node = None @@ -3016,10 +3040,11 @@ def get_subnodes_with_csts(self): check_existence=True) else: ndesc_list = [] - for n in sublist[1]: - shall_exist = self._existence_from_node(n[0]) + for ndesc in sublist[1]: + n, _, _ = self._get_node_and_attrs_from(ndesc) + shall_exist = self._existence_from_node(n) if shall_exist is None or shall_exist: - ndesc_list.append(n) + ndesc_list.append(ndesc) node = random.choice(ndesc_list) if ndesc_list else None if node is None: continue @@ -3084,11 +3109,11 @@ def handle_encoding(list_to_enc): new_item1vt.make_private(forget_current_state=False) new_item2vt = copy.copy(item2.get_value_type()) new_item2vt.make_private(forget_current_state=False) - new_item1vt.extend_right(new_item2vt) + new_item1vt.extend_left(new_item2vt) new_item.import_value_type(new_item1vt) new_item.frozen_node = new_item.get_value_type().get_current_value() if i > 0: - new_list = list_to_enc[:i-1] + new_list = list_to_enc[:i] new_list.append(new_item) if i < list_sz-2: new_list += list_to_enc[i+2:] @@ -3730,21 +3755,122 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, postponed_node_desc = None first_pass = True + + if self.custo.collapse_padding_mode: + consumed_bits = 0 + byte_aligned = None + # Iterate over all sub-components of the component node_list for delim, sublist in self.__iter_csts(node_list): if delim[1] == '>': - for node_desc in sublist: + for idx, node_desc in enumerate(sublist): abort = False base_node, min_node, max_node = self._parse_node_desc(node_desc) - if base_node.is_attr_set(NodeInternals.Abs_Postpone): + if self.custo.collapse_padding_mode: + + vt = base_node.get_value_type() + if not isinstance(vt, fvt.BitField) \ + or min_node != 1 \ + or max_node != 1 \ + or self.separator is not None \ + or postponed_node_desc: + raise DataModelDefinitionError('Pattern not supported for absorption') + + if not vt.lsb_padding or vt.endian != fvt.VT.BigEndian: + raise DataModelDefinitionError('Bitfield option not supported for ' + 'absorption with CollapsePadding custo') + + bytelen = vt.byte_length + if vt.padding_size != 0 or consumed_bits != 0: + last_idx = consumed_size+(bytelen-1) + + if consumed_bits != 0: + byte_aligned = consumed_bits + vt.padding_size == 8 + + bits_to_be_consumed = consumed_bits + vt.bit_length + last_idx = consumed_size + int(math.ceil(bits_to_be_consumed/8.0)) + + partial_blob = blob[consumed_size:last_idx] + if partial_blob != b'': + nb_bytes = len(partial_blob) + values = list(struct.unpack('B'*nb_bytes, partial_blob)) + result = 0 + for i, v in enumerate(values[::-1]): # big endian + if i == len(values)-1: + v = v & fvt.BitField.padding_one[8 - consumed_bits] + result += v<<(i*8) + + bits_to_consume = consumed_bits+vt.bit_length + mask_size = int(math.ceil(bits_to_consume/8.0))*8-bits_to_consume + + if not byte_aligned: + if vt.padding == 0: + result = result >> mask_size << mask_size + else: + result |= fvt.BitField.padding_one[mask_size] + + result <<= consumed_bits + if vt.padding == 1: + result |= fvt.BitField.padding_one[consumed_bits] + + l = [] + for i in range(nb_bytes-1, -1, -1): # big-endian + bval = result // (1 << i*8) + result = result % (1 << i*8) # remainder + l.append(bval) + if sys.version_info[0] > 2: + partial_blob = struct.pack('{:d}s'.format(nb_bytes), bytes(l)) + else: + partial_blob = struct.pack('{:d}s'.format(nb_bytes), str(bytearray(l))) + else: + partial_blob = blob[consumed_size:last_idx] + last_byte = blob[last_idx:last_idx+1] + if last_byte != b'': + val = struct.unpack('B', last_byte)[0] + if vt.padding == 0: + val = val >> vt.padding_size << vt.padding_size + else: + val |= fvt.BitField.padding_one[vt.padding_size] + partial_blob += struct.pack('B', val) + byte_aligned = False + else: + byte_aligned = True + else: + partial_blob = blob[consumed_size:consumed_size+bytelen] + byte_aligned = True + + abort, remaining_blob, consumed_size, consumed_nb, postponed_sent_back = \ + _try_absorption_with(base_node, 1, 1, + partial_blob, consumed_size, + None, + pending_upper_postpone=pending_postpone_desc) + + if partial_blob == b'' and abort is not None: + abort = True + break + + elif abort is not None and not abort: + consumed_bits = consumed_bits + vt.bit_length + consumed_bits = 0 if consumed_bits == 8 else consumed_bits%8 + + # if vt is byte-aligned, then the consumed_size is correct + if vt.padding_size != 0 and consumed_bits > 0: + consumed_size -= 1 + + if idx == len(sublist) - 1: + blob = blob[consumed_size:] + + elif base_node.is_attr_set(NodeInternals.Abs_Postpone): if postponed_node_desc or pending_postpone_desc: - raise ValueError("\nERROR: Only one node at a time (current:%s) delaying" \ - " its dissection is supported!" % postponed_node_desc) + raise ValueError("\n*** ERROR: Only one node at a time can have its " + "absorption delayed [current:{!s}]" + .format(postponed_node_desc)) postponed_node_desc = node_desc continue + else: # pending_upper_postpone = pending_postpone_desc abort, blob, consumed_size, consumed_nb, postponed_sent_back = \ @@ -3753,21 +3879,20 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, postponed_node_desc, pending_upper_postpone=pending_postpone_desc) - # In this case max_node is 0 - if abort is None: - continue + # In this case max_node is 0 + if abort is None: + continue - # if _try_absorption_with() return a - # tuple, then the postponed node is - # handled (either because absorption - # succeeded or because it didn't work and - # we need to abort and try another high - # level component) - if postponed_sent_back is not None: - postponed_to_send_back = postponed_sent_back - postponed_node_desc = None - pending_postpone_desc = None - # pending_upper_postpone = None + # if _try_absorption_with() return a + # tuple, then the postponed node is + # handled (either because absorption + # succeeded or because it didn't work and + # we need to abort and try another high + # level component) + if postponed_sent_back is not None: + postponed_to_send_back = postponed_sent_back + postponed_node_desc = None + pending_postpone_desc = None if abort: break @@ -4053,7 +4178,7 @@ def unfreeze(self, conf=None, recursive=True, dont_change_state=False, ignore_en excluded_idx=self.excluded_components, seed=self.component_seed) - fresh_expanded_nodelist = self._generate_expanded_nodelist(node_list) + fresh_expanded_nodelist = self._generate_expanded_nodelist(node_list, determinist=determinist) if self.expanded_nodelist is None: self.expanded_nodelist_origsz = len(fresh_expanded_nodelist) if self.expanded_nodelist_sz is not None: @@ -4083,12 +4208,13 @@ def unfreeze(self, conf=None, recursive=True, dont_change_state=False, ignore_en # don't bother trying to recover the previous one pass - for e in iterable: - self._cleanup_entangled_nodes_from(e) - if e.is_frozen(conf) and (e.is_nonterm(conf) or e.is_genfunc(conf) or e.is_func(conf)): - e.unfreeze(conf=conf, recursive=True, dont_change_state=dont_change_state, - ignore_entanglement=ignore_entanglement, only_generators=only_generators, - reevaluate_constraints=reevaluate_constraints) + if iterable is not None: + for n in iterable: + self._cleanup_entangled_nodes_from(n) + if n.is_nonterm(conf) or n.is_genfunc(conf) or n.is_func(conf): + n.unfreeze(conf=conf, recursive=True, dont_change_state=dont_change_state, + ignore_entanglement=ignore_entanglement, only_generators=only_generators, + reevaluate_constraints=reevaluate_constraints) self.frozen_node_list = None for n in self.subnodes_set: @@ -4101,12 +4227,11 @@ def unfreeze(self, conf=None, recursive=True, dont_change_state=False, ignore_en if self.separator is not None: iterable.add(self.separator.node) - if not reevaluate_constraints: - for e in iterable: - if e.is_frozen(conf): - e.unfreeze(conf=conf, recursive=True, dont_change_state=dont_change_state, - ignore_entanglement=ignore_entanglement, only_generators=only_generators, - reevaluate_constraints=reevaluate_constraints) + if not reevaluate_constraints and iterable is not None: + for n in iterable: + n.unfreeze(conf=conf, recursive=True, dont_change_state=dont_change_state, + ignore_entanglement=ignore_entanglement, only_generators=only_generators, + reevaluate_constraints=reevaluate_constraints) if not dont_change_state and not only_generators and not reevaluate_constraints: self._cleanup_entangled_nodes() @@ -4200,8 +4325,8 @@ def reset_fuzz_weight(self, recursive): for e in iterable: e.reset_fuzz_weight(recursive=recursive) - def set_child_env(self, env): + self.env = env iterable = copy.copy(self.subnodes_set) if self.separator is not None: iterable.add(self.separator.node) @@ -4680,6 +4805,9 @@ def __init__(self, name, base_node=None, copy_dico=None, ignore_frozen_state=Fal else: if new_env: self.env = Env() if ignore_frozen_state else copy.copy(base_node.env) + # we always keep a reference to objects coming from other part of the framework, + # namely: knowledge_source + # self.env.knowledge_source = base_node.env.knowledge_source else: self.env = base_node.env @@ -4820,6 +4948,7 @@ def set_contents(self, base_node, forget_original_sync_objs=False) self.internals[conf] = new_internals + self.internals[conf].env = self.env if base_node.is_nonterm(conf): self.internals[conf].import_subnodes_full_format(internals=base_node.internals[conf]) @@ -4833,9 +4962,6 @@ def set_contents(self, base_node, delayed_node_internals=delayed_node_internals) self._finalize_nonterm_node(conf) - elif base_node.is_genfunc(conf) or base_node.is_func(conf): - self.internals[conf].set_env(self.env) - # Once node_dico has been populated from the node tree, # we deal with 'nodes' argument of Func and GenFunc that does not belong to this # tree. And we complete the node_dico. @@ -5061,7 +5187,7 @@ def conf(self, conf=None): def get_internals_backup(self): return Node(self.name, base_node=self, ignore_frozen_state=False, - accept_external_entanglement=True) + accept_external_entanglement=True, new_env=False) def set_internals(self, backup): self.name = backup.name @@ -5614,15 +5740,11 @@ def get_all_paths_from(self, node, conf=None): l.append(n) return l - - def __set_env_rec(self, env): + def set_env(self, env): self.env = env for c in self.internals: self.internals[c].set_child_env(env) - def set_env(self, env): - self.__set_env_rec(env) - def get_env(self): return self.env @@ -5631,6 +5753,10 @@ def freeze(self, conf=None, recursive=True, return_node_internals=False): ret = self._get_value(conf=conf, recursive=recursive, return_node_internals=return_node_internals) + if self.env is None: + print('Warning: freeze() is called on a node that does not have an Env()\n' + ' --> node name: {!s}'.format(self.name)) + if self.env is not None and self.env.delayed_jobs_enabled and \ (not self._delayed_jobs_called or self.env.delayed_jobs_pending): self._delayed_jobs_called = True @@ -5791,10 +5917,10 @@ def unfreeze(self, conf=None, recursive=True, dont_change_state=False, if not self.is_conf_existing(conf): conf = self.current_conf - if self.is_frozen(conf): - self.internals[conf].unfreeze(next_conf, recursive=recursive, dont_change_state=dont_change_state, - ignore_entanglement=ignore_entanglement, only_generators=only_generators, - reevaluate_constraints=reevaluate_constraints) + # if self.is_frozen(conf): + self.internals[conf].unfreeze(next_conf, recursive=recursive, dont_change_state=dont_change_state, + ignore_entanglement=ignore_entanglement, only_generators=only_generators, + reevaluate_constraints=reevaluate_constraints) if not ignore_entanglement and self.entangled_nodes is not None: for e in self.entangled_nodes: @@ -6198,6 +6324,8 @@ def __copy__(self): class Env(object): + knowledge_source = None + def __init__(self): self.exhausted_nodes = [] self.nodes_to_corrupt = {} @@ -6209,7 +6337,7 @@ def __init__(self): self._dm = None self.id_list = None self._reentrancy_cpt = 0 - # self.cpt = 0 + # self._knowledge_source = None @property def delayed_jobs_pending(self): @@ -6230,6 +6358,14 @@ def set_data_model(self, dm): def get_data_model(self): return self._dm + # @property + # def knowledge_source(self): + # return self._knowledge_source + # + # @knowledge_source.setter + # def knowledge_source(self, src): + # self._knowledge_source = src + def add_node_to_corrupt(self, node, corrupt_type=None, corrupt_op=lambda x: x): if node.entangled_nodes: for n in node.entangled_nodes: diff --git a/framework/node_builder.py b/framework/node_builder.py index b85f7e3..e96f062 100644 --- a/framework/node_builder.py +++ b/framework/node_builder.py @@ -108,7 +108,7 @@ def create_graph_from_desc(self, desc): n = self._create_graph_from_desc(desc, None) if self._add_env_to_the_node: - self._register_todo(n, self._set_env, prio=self.LOW_PRIO) + self._register_todo(n, self._set_env, prio=self.VERYLOW_PRIO) todo = self._create_todo_list() while todo: @@ -204,7 +204,7 @@ def __handle_clone(self, desc, parent_node): if clone_ref is not None: ref = self._handle_name(clone_ref) self._register_todo(nd, self._clone_from_dict, args=(ref, desc), - prio=self.MEDIUM_PRIO) + prio=self.LOW_PRIO) self.node_dico[(name, ident)] = nd else: ref = (name, ident) @@ -218,7 +218,7 @@ def __handle_clone(self, desc, parent_node): return nd def __pre_handling(self, desc, node): - if node: + if node is not None: if isinstance(node.cc, NodeInternals_Empty): raise ValueError("Error: alternative configuration"\ " cannot be added to empty node ({:s})".format(node.name)) @@ -352,10 +352,14 @@ def _create_non_terminal_node(self, desc, node=None): if w is not None: # in this case there are multiple shapes, as shape can be # discriminated by its weight attr - for s in desc.get('contents'): + for s in cts: self._verify_keys_conformity(s) weight = s.get('weight', 1) - shape = self._create_nodes_from_shape(s['contents'], n) + subnodes = s['contents'] + shtype = s.get('shape_type', MH.Ordered) + dupmode = s.get('duplicate_mode', MH.Copy) + shape = self._create_nodes_from_shape(subnodes, n, shape_type=shtype, + dup_mode=dupmode) shapes.append(weight) shapes.append(shape) else: @@ -379,7 +383,7 @@ def _create_non_terminal_node(self, desc, node=None): prefix = sep_desc.get('prefix', True) suffix = sep_desc.get('suffix', True) unique = sep_desc.get('unique', False) - n.set_separator_node(sep_node, prefix=prefix, suffix=suffix, unique=unique) + n.conf(conf).set_separator_node(sep_node, prefix=prefix, suffix=suffix, unique=unique) self._handle_common_attr(n, desc, conf) @@ -521,7 +525,8 @@ def _handle_common_attr(self, node, desc, conf): vals = desc.get('specific_fuzzy_vals', None) if vals is not None: if not node.is_typed_value(conf=conf): - raise DataModelDefinitionError("'specific_fuzzy_vals' is only usable with Typed-nodes") + raise DataModelDefinitionError("'specific_fuzzy_vals' is only usable with Typed-nodes." + " [guilty node: '{:s}']".format(node.name)) node.conf(conf).set_specific_fuzzy_values(vals) param = desc.get('mutable', None) if param is not None: @@ -537,10 +542,16 @@ def _handle_common_attr(self, node, desc, conf): node.clear_attr(MH.Attr.DEBUG, conf=conf) param = desc.get('determinist', None) if param is not None: - node.make_determinist(conf=conf) + if param: + node.make_determinist(conf=conf) + else: + node.make_random(conf=conf) param = desc.get('random', None) if param is not None: - node.make_random(conf=conf) + if param: + node.make_random(conf=conf) + else: + node.make_determinist(conf=conf) param = desc.get('finite', None) if param is not None: node.make_finite(conf=conf) @@ -622,7 +633,7 @@ def _handle_common_attr(self, node, desc, conf): if encoder is not None: node.set_encoder(encoder) - def _register_todo(self, node, func, args=None, unpack_args=True, prio=VERYLOW_PRIO): + def _register_todo(self, node, func, args=None, unpack_args=True, prio=MEDIUM_PRIO): if self.sorted_todo.get(prio, None) is None: self.sorted_todo[prio] = [] self.sorted_todo[prio].insert(0, (node, func, args, unpack_args)) @@ -1364,6 +1375,10 @@ def flush(self): if self.values is not None and all(val.isdigit() for val in self.values): self.values = [int(i) for i in self.values] type = fvt.INT_str + elif self.alphabet is not None and all(c.isdigit() for c in self.alphabet): + self.values = [int(c) for c in self.alphabet] + type = fvt.INT_str + self.alphabet = None else: type = fvt.String diff --git a/framework/operator_helpers.py b/framework/operator_helpers.py index 56e7005..9b38606 100644 --- a/framework/operator_helpers.py +++ b/framework/operator_helpers.py @@ -56,9 +56,9 @@ def is_flag_set(self, name): def set_status(self, status): self.status = status - def add_instruction(self, actions, seed=None): + def add_instruction(self, actions, seed=None, tg_ids=None): l = list(actions) if actions is not None else None - self.action_register.append((l, seed)) + self.action_register.append((l, seed, tg_ids)) def get_instructions(self): return self.action_register @@ -72,7 +72,7 @@ def __init__(self): self._now = datetime.datetime.now() self.comments = None self.feedback_info = None - self._status_code = None + self._status_code = 0 self.instructions = { LastInstruction.RecordData: False } @@ -111,8 +111,10 @@ def get_timestamp(self): class Operator(object): - def __init__(self): - pass + _args_desc = None + + def __str__(self): + return "Operator '{:s}'".format(self.__class__.__name__) def start(self, fmk_ops, dm, monitor, target, logger, user_input): ''' @@ -159,8 +161,8 @@ def do_after_all(self, fmk_ops, dm, monitor, target, logger): return linst def _start(self, fmk_ops, dm, monitor, target, logger, user_input): - sys.stdout.write("\n__ setup operator '%s' __" % self.__class__.__name__) - if not _user_input_conformity(self, user_input, self._gen_args_desc, self._args_desc): + # sys.stdout.write("\n__ setup operator '%s' __" % self.__class__.__name__) + if not _user_input_conformity(self, user_input, self._args_desc): return False _handle_user_inputs(self, user_input) @@ -176,25 +178,17 @@ def _start(self, fmk_ops, dm, monitor, target, logger, user_input): return ok -def operator(prj, gen_args={}, args={}): +def operator(prj, args=None): def internal_func(operator_cls): - operator_cls._gen_args_desc = gen_args - operator_cls._args_desc = args - # check conflict between gen_args & args - for k in gen_args: - if k in args.keys(): - raise ValueError("Specific parameter '{:s}' is in conflict with a generic parameter!".format(k)) - # create generic attributes - for k, v in gen_args.items(): - desc, default, arg_type = v - setattr(operator_cls, k, default) - # create specific attributes - for k, v in args.items(): - desc, default, arg_type = v - setattr(operator_cls, k, default) + operator_cls._args_desc = {} if args is None else args + if args is not None: + # create specific attributes + for k, v in args.items(): + desc, default, arg_type = v + setattr(operator_cls, k, default) # register an object of this class operator = operator_cls() - prj.register_new_operator(operator.__class__.__name__, operator) + prj.register_operator(operator.__class__.__name__, operator) return operator_cls return internal_func diff --git a/framework/plumbing.py b/framework/plumbing.py index e8a1ae1..f7a2516 100644 --- a/framework/plumbing.py +++ b/framework/plumbing.py @@ -38,12 +38,8 @@ import time import signal -from libs.external_modules import * - -from framework.node import * -from framework.data import * -from framework.data_model import DataModel -from framework.database import FeedbackHandler +from framework.database import FeedbackGate +from framework.knowledge.feedback_collector import FeedbackSource from framework.error_handling import * from framework.evolutionary_helpers import EvolutionaryScenariosFactory from framework.logger import * @@ -55,12 +51,14 @@ from framework.target_helpers import * from framework.targets.local import LocalTarget from framework.targets.printer import PrinterTarget +from framework.cosmetics import aligned_stdout +from framework.config import config, config_dot_proxy from libs.utils import * import framework.generic_data_makers -import data_models -import projects +import data_models # needed by importlib.reload +import projects # needed by importlib.reload from framework.global_resources import * from libs.utils import * @@ -70,8 +68,11 @@ user_dm_mod = os.path.basename(os.path.normpath(user_data_models_folder)) user_prj_mod = os.path.basename(os.path.normpath(user_projects_folder)) +user_tg_mod = os.path.basename(os.path.normpath(user_targets_folder)) + exec('import ' + user_dm_mod) exec('import ' + user_prj_mod) +exec('import ' + user_tg_mod) sig_int_handler = signal.getsignal(signal.SIGINT) @@ -192,7 +193,11 @@ def run(self): while not self._stop.is_set(): try: # print("\n*** Function '{!s}' executed by Task '{!s}' ***".format(self._func, self._name)) - self._func(self._arg) + if isinstance(self._func, list): + for f in self._func: + f(self._arg) + else: + self._func(self._arg) except DataProcessTermination: break except: @@ -214,37 +219,59 @@ class FmkPlumbing(object): Defines the methods to operate every sub-systems of fuddly ''' - def __init__(self): + def __init__(self, exit_on_error=False, debug_mode=False): + self._debug_mode = debug_mode + + self._prj = None + self.dm = None + self.lg = None + + self.targets = {} # enabled targets, further initialized as a dict (tg_id -> tg obj) + self._tg_ids = [0] # further initialized as a list + self.available_targets_desc = None # further initialized as a dict (tg -> str description) + self._currently_used_targets = [] + + self.mon = None + + self.prj_list = [] + self.dm_list = [] + + self._hc_timeout = {} # health-check tiemout, further initialized as a dict (tg -> hc_timeout) + self._hc_timeout_max = None + self.__started = False self.__first_loading = True + self._current_sent_date = None + self.error = False self.fmk_error = [] self._sending_error = None + self._stop_sending = None self.__tg_enabled = False self.__prj_to_be_reloaded = False + self._dm_to_be_reloaded = False self._exportable_fmk_ops = ExportableFMKOps(self) self._generic_tactics = framework.generic_data_makers.tactics self._generic_tactics.set_exportable_fmk_ops(self._exportable_fmk_ops) + self._tactics = None + self.import_text_reg = re.compile('(.*?)(#####)', re.S) self.check_clone_re = re.compile('(.*)#(\w{1,20})') - self.prj_list = [] self._prj_dict = {} - - self.dm_list = [] self.__st_dict = {} self.__target_dict = {} - self.__current_tg = 0 self.__logger_dict = {} self.__monitor_dict = {} self.__initialized_dmaker_dict = {} self.__dm_rld_args_dict= {} self.__prj_rld_args_dict= {} + self.__initialized_dmakers = None self.__dyngenerators_created = {} self.__dynamic_generator_ids = {} @@ -255,21 +282,39 @@ def __init__(self): self._task_list = {} self._task_list_lock = threading.Lock() + self.config = config(self, path=[config_folder]) + def save_config(): + filename=os.path.join( + config_folder, + self.config.config_name + '.ini') + with open(filename, 'w') as cfile: + self.config.write(cfile) + atexit.register(save_config) + self.fmkDB = Database() ok = self.fmkDB.start() if not ok: raise InvalidFmkDB("The database {:s} is invalid!".format(self.fmkDB.fmk_db_path)) - self.feedback_handler = FeedbackHandler(self.fmkDB) + self.feedback_gate = FeedbackGate(self.fmkDB) + Project.feedback_gate = self.feedback_gate self._fmkDB_insert_dm_and_dmakers('generic', self._generic_tactics) self.group_id = 0 - self._saved_group_id = None # used by self._recover_target() + self._recovered_tgs = None # used by self._recover_target() self.enable_wkspace() + self.import_successfull = True self.get_data_models() + if exit_on_error and not self.import_successfull: + self.fmkDB.stop() + raise DataModelDefinitionError('Error with some DM imports') + self.get_projects() + if exit_on_error and not self.import_successfull: + self.fmkDB.stop() + raise ProjectDefinitionError('Error with some Project imports') print(colorize(FontStyle.BOLD + '='*44 + '[ Fuddly Data Folder Information ]==\n', rgb=Color.FMKINFOGROUP)) @@ -285,10 +330,22 @@ def __init__(self): ' - user projects and user data models', rgb=Color.FMKSUBINFO)) + def __str__(self): + return 'Fuddly FmK' + + @property + def prj(self): + return self._prj + + @prj.setter + def prj(self, obj): + self._prj = obj + self.fmkDB.current_project = obj + def set_error(self, msg='', context=None, code=Error.Reserved): self.error = True self.fmk_error.append(Error(msg, context=context, code=code)) - if hasattr(self, 'lg'): + if self.lg: self.lg.log_fmk_info(msg) def get_error(self): @@ -311,29 +368,30 @@ def __reset_fmk_internals(self, reset_existing_seed=True): self.cleanup_all_dmakers(reset_existing_seed) # Warning: fuzz delay is not set to 0 by default in order to have a time frame # where SIGINT is accepted from user - self.set_fuzz_delay(0.01) - self.set_fuzz_burst(1) - self._recompute_health_check_timeout(self.tg.feedback_timeout, self.tg.sending_delay) + self.set_fuzz_delay(self.config.defvalues.fuzz.delay) + self.set_fuzz_burst(self.config.defvalues.fuzz.burst) + for tg in self.targets.values(): + self._recompute_health_check_timeout(tg.feedback_timeout, tg.sending_delay, target=tg) - def _recompute_health_check_timeout(self, base_timeout, sending_delay, do_show=True): + def _recompute_health_check_timeout(self, base_timeout, sending_delay, target=None, do_show=True): if base_timeout is not None: if base_timeout != 0: if 0 < base_timeout < 1: hc_timeout = base_timeout + sending_delay + 0.5 else: hc_timeout = base_timeout + sending_delay + 2.0 - self.set_health_check_timeout(hc_timeout, do_show=do_show) + self.set_health_check_timeout(hc_timeout, target=target, do_show=do_show) else: # base_timeout comes from feedback_timeout, if it is equal to 0 # this is a special meaning used internally to collect residual feedback. # Thus, we don't change the current health_check timeout. return else: - self.set_health_check_timeout(max(10,sending_delay), do_show=do_show) + self.set_health_check_timeout(max(10,sending_delay), target=target, do_show=do_show) def _handle_user_code_exception(self, msg='', context=None): self.set_error(msg, code=Error.UserCodeError, context=context) - if hasattr(self, 'lg'): + if self.lg: self.lg.log_error("Exception in user code detected! Outcomes " \ "of this log entry has to be considered with caution.\n" \ " (_ cause: '%s' _)" % msg) @@ -344,7 +402,7 @@ def _handle_user_code_exception(self, msg='', context=None): def _handle_fmk_exception(self, cause=''): self.set_error(cause, code=Error.UserCodeError) - if hasattr(self, 'lg'): + if self.lg: self.lg.log_error("Not handled exception detected! Outcomes " \ "of this log entry has to be considered with caution.\n" \ " (_ cause: '%s' _)" % cause) @@ -379,47 +437,83 @@ def is_valid(d): @EnforceOrder(accepted_states=['S2']) def reload_dm(self): - prefix = self.__dm_rld_args_dict[self.dm][0] - name = self.__dm_rld_args_dict[self.dm][1] + return self._reload_dm() + + @EnforceOrder(always_callable=True, transition=['25_load_dm','S1']) + def _reload_dm(self, dm_name=None): + if dm_name is None: + prefix = self.__dm_rld_args_dict[self.dm][0] + name = self.__dm_rld_args_dict[self.dm][1] + else: + if isinstance(dm_name, list): + prefix = None + name = dm_name + else: + dm_obj = self.get_data_model_by_name(dm_name) + prefix = self.__dm_rld_args_dict[dm_obj][0] + name = dm_name if prefix is None: # In this case we face a composed DM, name is in fact a dm_list dm_list = name name_list = [] - self.cleanup_all_dmakers() + self._cleanup_all_dmakers() - for dm in dm_list: - name_list.append(dm.name) - self.dm = dm - self.reload_dm() + orig_dm = self.dm + for idx, dm_ref in enumerate(copy.copy(dm_list)): + if isinstance(dm_ref, str): + dm_name = dm_ref + dm_obj = self.get_data_model_by_name(dm_ref) + else: + dm_name = dm_ref.name + dm_obj = dm_ref + + name_list.append(dm_name) + self.dm = dm_obj + ok = self._reload_dm() + dm_list[idx] = self.dm + + if not ok: + self.dm = orig_dm + return False # reloading is based on name because DM objects have changed if not self.load_multiple_data_model(name_list=name_list, reload_dm=True): self.set_error("Error encountered while reloading the composed Data Model") else: - self.cleanup_all_dmakers() + self._cleanup_all_dmakers() self.dm.cleanup() dm_params = self.__import_dm(prefix, name, reload_dm=True) if dm_params is not None: + if self.dm in self.__dynamic_generator_ids: + del self.__dynamic_generator_ids[self.dm] + if self.dm in self.__dyngenerators_created: + del self.__dyngenerators_created[self.dm] self.__add_data_model(dm_params['dm'], dm_params['tactics'], dm_params['dm_rld_args'], reload_dm=True) self.__dyngenerators_created[dm_params['dm']] = False + try: + i = self.dm_list.index(self.dm) + except ValueError: + pass + else: + self.dm_list[i] = dm_params['dm'] self.dm = dm_params['dm'] else: return False self._cleanup_dm_attrs_from_fmk() - - if not self._load_data_model(): - return False + if self._is_started(): + if not self._load_data_model(): + return False self.prj.set_data_model(self.dm) - if hasattr(self, 'tg'): - self.tg.set_data_model(self.dm) - if hasattr(self, 'mon'): + for tg in self.targets.values(): + tg.set_data_model(self.dm) + if self.mon: self.mon.set_data_model(self.dm) self._fmkDB_insert_dm_and_dmakers(self.dm.name, dm_params['tactics']) @@ -428,7 +522,7 @@ def reload_dm(self): def _cleanup_dm_attrs_from_fmk(self): self._generic_tactics.clear_generator_clones() self._generic_tactics.clear_disruptor_clones() - if hasattr(self, '_tactics'): + if self._tactics: self._tactics.clear_generator_clones() self._tactics.clear_disruptor_clones() self._tactics = self.__st_dict[self.dm] @@ -436,20 +530,20 @@ def _cleanup_dm_attrs_from_fmk(self): @EnforceOrder(accepted_states=['S2']) - def reload_all(self, tg_num=None): - return self.__reload_all(tg_num=tg_num) + def reload_all(self, tg_ids=None): + return self._reload_all(tg_ids=tg_ids) - def __reload_all(self, tg_num=None): + def _reload_all(self, tg_ids=None): prj_prefix = self.__prj_rld_args_dict[self.prj][0] prj_name = self.__prj_rld_args_dict[self.prj][1] dm_prefix = self.__dm_rld_args_dict[self.dm][0] dm_name = self.__dm_rld_args_dict[self.dm][1] - self.__stop_fmk_plumbing() + self._stop_fmk_plumbing() - if tg_num is not None: - self.set_target(tg_num) + if tg_ids is not None: + self.load_targets(tg_ids) prj_params = self._import_project(prj_prefix, prj_name, reload_prj=True) if prj_params is not None: @@ -459,24 +553,24 @@ def __reload_all(self, tg_num=None): if dm_prefix is None: # it is ok to call reload_dm() here because it is a # composed DM, and it won't call the methods used within - # __init_fmk_internals_step1(). - self.reload_dm() - self.__init_fmk_internals_step1(prj_params['project'], self.dm) + # _init_fmk_internals_step1(). + self._reload_dm() + self._init_fmk_internals_step1(prj_params['project'], self.dm) else: dm_params = self.__import_dm(dm_prefix, dm_name, reload_dm=True) if dm_params is not None: self.__add_data_model(dm_params['dm'], dm_params['tactics'], dm_params['dm_rld_args'], reload_dm=True) self.__dyngenerators_created[dm_params['dm']] = False - self.__init_fmk_internals_step1(prj_params['project'], dm_params['dm']) + self._init_fmk_internals_step1(prj_params['project'], dm_params['dm']) - self.__start_fmk_plumbing() + self._start_fmk_plumbing() if self.is_not_ok(): - self.__stop_fmk_plumbing() + self._stop_fmk_plumbing() return False if prj_params is not None: - self.__init_fmk_internals_step2(prj_params['project'], self.dm) + self._init_fmk_internals_step2(prj_params['project'], self.dm) return True @@ -499,17 +593,20 @@ def _fmkDB_insert_dm_and_dmakers(self, dm_name, tactics): gen_obj = tactics.get_generator_obj(gen_type, gen_name) self.fmkDB.insert_dmaker(dm_name, gen_type, gen_name, True, True) - def _recover_target(self): - if self.group_id == self._saved_group_id: + def _recover_target(self, tg): + if self._recovered_tgs and tg in self._recovered_tgs: # This method can be called after checking target health, feedback and # probes status. However, we have to avoid to recover the target twice. return True else: - self._saved_group_id = self.group_id + if self._recovered_tgs is None: + self._recovered_tgs = {tg} + else: + self._recovered_tgs.add(tg) target_recovered = False try: - target_recovered = self.tg.recover_target() + target_recovered = tg.recover_target() except NotImplementedError: self.lg.log_fmk_info("No method to recover the target is implemented! (assumption: no need " "to recover)") @@ -517,37 +614,40 @@ def _recover_target(self): except: self.lg.log_fmk_info("Exception raised while trying to recover the target!") else: + tg_desc = self.available_targets_desc[tg] if target_recovered: - self.lg.log_fmk_info("The target has been recovered!") + self.lg.log_fmk_info("The target {!s} has been recovered!".format(tg_desc)) else: - self.lg.log_fmk_info("The target has not been recovered! All further operations " - "will be terminated.") + self.lg.log_fmk_info("The target {!s} has not been recovered! All further operations " + "will be terminated.".format(tg_desc)) return target_recovered def monitor_probes(self, prefix=None, force_record=False): - probes = self.mon.get_probes_names() - ok = True + oks = {x: True for x in self.targets.values()} prefix_printed = False - for pname in probes: - if self.mon.is_probe_launched(pname): - pstatus = self.mon.get_probe_status(pname) - err = pstatus.get_status() + + for probe in self.mon.iter_probes(): + if self.mon.is_probe_launched(probe): + pstatus = self.mon.get_probe_status(probe) + err = pstatus.value if err < 0 or force_record: + tg = self.mon.get_probe_related_tg(probe) if err < 0: - ok = False + if tg is not None: + oks[tg] = False if prefix and not prefix_printed: prefix_printed = True self.lg.print_console('\n*** {:s} ***'.format(prefix), rgb=Color.FMKINFO) tstamp = pstatus.get_timestamp() priv = pstatus.get_private_info() - self.lg.log_probe_feedback(source="Probe '{:s}'".format(pname), - timestamp=tstamp, - content=priv, status_code=err) + self.lg.log_probe_feedback(probe=probe, content=priv, status_code=err, + timestamp=tstamp, related_tg=tg) - ret = self._recover_target() if not ok else True + for tg, ok in oks.items(): + ret = self._recover_target(tg) if not ok else True - if prefix and not ok: - self.lg.print_console('*'*(len(prefix)+8)+'\n', rgb=Color.FMKINFO) + if prefix and not ok: + self.lg.print_console('*'*(len(prefix)+8)+'\n', rgb=Color.FMKINFO) return ret @@ -603,12 +703,13 @@ def populate_data_models(path): self.__dyngenerators_created[dm_params['dm']] = False # populate FMK DB self._fmkDB_insert_dm_and_dmakers(dm_params['dm'].name, dm_params['tactics']) + else: + self.import_successfull = False self.fmkDB.insert_data_model(Database.DEFAULT_DM_NAME) self.fmkDB.insert_dmaker(Database.DEFAULT_DM_NAME, Database.DEFAULT_GTYPE_NAME, Database.DEFAULT_GEN_NAME, True, True) - def __import_dm(self, prefix, name, reload_dm=False): try: @@ -697,10 +798,8 @@ def __add_data_model(self, data_model, strategy, dm_rld_args, if old_dm is not None: self.__dm_rld_args_dict.pop(old_dm) self.__st_dict.pop(old_dm) - self.__st_dict[data_model] = strategy - else: - self.__st_dict[data_model] = strategy + self.__st_dict[data_model] = strategy self.__dm_rld_args_dict[data_model] = dm_rld_args @@ -751,11 +850,12 @@ def populate_projects(path): name = res.group(1) prj_params = self._import_project(prefix, name) if prj_params is not None: - self._add_project(prj_params['project'], - prj_params['target'], prj_params['logger'], - prj_params['prj_rld_args'], + self._add_project(prj_params['project'], prj_params['target'], + prj_params['logger'], prj_params['prj_rld_args'], reload_prj=False) self.fmkDB.insert_project(prj_params['project'].name) + else: + self.import_successfull = False def _import_project(self, prefix, name, reload_prj=False): @@ -774,7 +874,6 @@ def _import_project(self, prefix, name, reload_prj=False): print(colorize("*** Problem during reload of '%s_proj.py' ***" % (name), rgb=Color.ERROR)) else: print(colorize("*** Problem during import of '%s_proj.py' ***" % (name), rgb=Color.ERROR)) - print(prefix) print('-'*60) traceback.print_exc(file=sys.stdout) print('-'*60) @@ -795,7 +894,7 @@ def _import_project(self, prefix, name, reload_prj=False): try: logger = eval(prefix + name + '_proj' + '.logger') except: - logger = Logger(name, prefix=' || ') + logger = Logger(name) logger.fmkDB = self.fmkDB if logger.name is None: logger.name = name @@ -821,8 +920,11 @@ def _import_project(self, prefix, name, reload_prj=False): new_targets.append(tg) targets = new_targets - if self.__current_tg >= len(targets): - self.__current_tg = 0 + for idx, tg_id in enumerate(self._tg_ids): + if tg_id >= len(targets): + print(colorize("*** Incorrect Target ID detected: {:d} --> replace with 0 ***".format(tg_id), + rgb=Color.WARNING)) + self._tg_ids[idx] = 0 prj_params['target'] = targets @@ -838,8 +940,7 @@ def _import_project(self, prefix, name, reload_prj=False): return prj_params - def _add_project(self, project, target, logger, prj_rld_args, - reload_prj=False): + def _add_project(self, project, targets, logger, prj_rld_args, reload_prj=False): if project.name not in map(lambda x: x.name, self.prj_list): self.prj_list.append(project) @@ -864,48 +965,57 @@ def _add_project(self, project, target, logger, prj_rld_args, mon = self.__monitor_dict.pop(old_prj) lg = self.__logger_dict.pop(old_prj) tg = self.__target_dict.pop(old_prj) - self.__target_dict[project] = target + self.__target_dict[project] = targets self.__logger_dict[project] = logger self.__monitor_dict[project] = project.monitor self.__monitor_dict[project].set_fmk_ops(fmk_ops=self._exportable_fmk_ops) self.__monitor_dict[project].set_logger(self.__logger_dict[project]) - self.__monitor_dict[project].set_target(self.__target_dict[project]) + # self.__monitor_dict[project].set_targets(self.__target_dict[project]) self._prj_dict[project].set_logger(self.__logger_dict[project]) self._prj_dict[project].set_monitor(self.__monitor_dict[project]) else: self._prj_dict[project] = project - self.__target_dict[project] = target + self.__target_dict[project] = targets self.__logger_dict[project] = logger self.__monitor_dict[project] = project.monitor self.__monitor_dict[project].set_fmk_ops(fmk_ops=self._exportable_fmk_ops) self.__monitor_dict[project].set_logger(self.__logger_dict[project]) - self.__monitor_dict[project].set_target(self.__target_dict[project]) + # self.__monitor_dict[project].set_target(self.__target_dict[project]) self._prj_dict[project].set_logger(self.__logger_dict[project]) self._prj_dict[project].set_monitor(self.__monitor_dict[project]) self.__prj_rld_args_dict[project] = prj_rld_args self.__initialized_dmaker_dict[project] = {} - - - def is_usable(self): - return self.__is_started() + return self._is_started() - def __is_started(self): + def _is_started(self): return self.__started - def __start(self): + def _start(self): self.__started = True - def __stop(self): + def _stop(self): self.__started = False def _load_data_model(self): try: self.dm.load_data_model(self._name2dm) + except: + msg = "Error encountered while loading the data model. (checkup the associated" \ + " '{:s}.py' file)".format(self.dm.name) + self._handle_user_code_exception(msg=msg) + self._dm_to_be_reloaded = True + if self.dm in self.__dyngenerators_created: + del self.__dyngenerators_created[self.dm] + if self.dm in self.__dynamic_generator_ids: + del self.__dynamic_generator_ids[self.dm] + return False + + else: if not self.__dyngenerators_created[self.dm]: self.__dyngenerators_created[self.dm] = True self.__dynamic_generator_ids[self.dm] = [] @@ -920,18 +1030,12 @@ def _load_data_model(self): self.fmkDB.insert_dmaker(self.dm.name, dmaker_type, gen_cls_name, True, True) print(colorize("*** Data Model '%s' loaded ***" % self.dm.name, rgb=Color.DATA_MODEL_LOADED)) - - except: - self._handle_user_code_exception() - self.__prj_to_be_reloaded = True - self.set_error("Error encountered while loading the data model. (checkup" \ - " the associated '%s.py' file)" % self.dm.name) - return False + self._dm_to_be_reloaded = False return True - def __start_fmk_plumbing(self): - if not self.__is_started(): + def _start_fmk_plumbing(self): + if not self._is_started(): signal.signal(signal.SIGINT, signal.SIG_IGN) self.lg.start() @@ -941,39 +1045,52 @@ def __start_fmk_plumbing(self): self.set_error("Project cannot be launched because of data model loading error") return + ok = {} try: - ok = self.tg._start() + for tg_id, tg in self.targets.items(): + ok[tg_id] = tg._start(self.available_targets_desc[tg], tg_id) except: self._handle_user_code_exception() - self.set_error("The Target has not been initialized correctly (checkup" \ - " the associated '%s_strategy.py' file)" % self.dm.name) + self.set_error("The Target {!s} has not been initialized correctly" + .format(self.available_targets_desc[tg])) else: - if ok: - self.__enable_target() - self.mon.start() - for p in self.tg.probes: - pname, delay = self._extract_info_from_probe(p) + for tg_id in self.targets: + if not ok[tg_id]: + self.set_error("The Target has not been initialized correctly") + return + + self._enable_target() + self.mon.start() + + need_monitoring = False + for tg in self.targets.values(): + if tg.probes: + need_monitoring = True + + for p in tg.probes: + pobj, delay = self._extract_info_from_probe(p) if delay is not None: - self.mon.set_probe_delay(pname, delay) - self.mon.start_probe(pname) - self.mon.wait_for_probe_initialization() - self.prj.start() - if self.tg.probes: - time.sleep(0.5) - self.monitor_probes(force_record=True) - else: - self.set_error("The Target has not been initialized correctly") - - self.__current = [] - self.__db_idx = 0 - self.__data_bank = {} + self.mon.set_probe_delay(pobj, delay) + self.mon.start_probe(pobj, related_tg=tg) + + self.mon.wait_for_probe_initialization() + self.prj.start() - self.__start() + if need_monitoring: + time.sleep(0.5) + self.monitor_probes(force_record=True) + finally: + self.__current = [] + self.__db_idx = 0 + self.__data_bank = {} + + self._start() - def __stop_fmk_plumbing(self): + + def _stop_fmk_plumbing(self): self.flush_errors() - if self.__is_started(): + if self._is_started(): if self.is_target_enabled(): self.log_target_residual_feedback() @@ -982,78 +1099,60 @@ def __stop_fmk_plumbing(self): if self.is_target_enabled(): self.mon.stop() try: - self.tg._stop() + for tg_id, tg in self.targets.items(): + tg._stop(self.available_targets_desc[tg], tg_id) except: self._handle_user_code_exception() finally: - self.__disable_target() + self._disable_target() self.lg.stop() self.prj.stop() - self.__stop() + self._stop() signal.signal(signal.SIGINT, sig_int_handler) @EnforceOrder(accepted_states=['20_load_prj','25_load_dm','S1','S2']) def exit_fmk(self): - self.__stop_fmk_plumbing() + self._stop_fmk_plumbing() self.fmkDB.stop() @EnforceOrder(accepted_states=['25_load_dm','S1','S2']) - def set_target(self, num): - return self.__set_target(num) + def load_targets(self, tg_ids): + return self._load_targets(tg_ids) - def __set_target(self, num): - if num >= len(self.__target_dict[self.prj]): - self.set_error('The provided target number does not exist!', - code=Error.CommandError) - return False + def _load_targets(self, tg_ids): + for tg_id in tg_ids: + if tg_id >= len(self.__target_dict[self.prj]): + self.set_error('The provided target number does not exist!', + code=Error.CommandError) + return False + + self._tg_ids = tg_ids - self.__current_tg = num return True @EnforceOrder(accepted_states=['25_load_dm','S1','S2']) def get_available_targets(self): - for tg in self.__target_dict[self.prj]: - yield tg + return self.__target_dict[self.prj] def _extract_info_from_probe(self, p): if isinstance(p, (tuple, list)): assert(len(p) == 2) - pname = p[0].__name__ + pobj = p[0] delay = p[1] else: - pname = p.__name__ + pobj = p delay = None - return pname, delay + return pobj, delay def _get_detailed_target_desc(self, tg): - if isinstance(tg, PrinterTarget): - printer_name = tg.get_printer_name() - printer_name = ', Name: ' + printer_name if printer_name is not None else '' - detailed_desc = tg.__class__.__name__ + ' [IP: ' + tg.get_target_ip() + printer_name + ']' - elif isinstance(tg, LocalTarget): - pre_args = tg.get_pre_args() - post_args = tg.get_post_args() - args = '' - if pre_args or post_args: - if pre_args is not None: - args += pre_args - if post_args is not None: - args += post_args - args = ', Args: ' + args - detailed_desc = tg.__class__.__name__ + ' [Program: ' + tg.get_target_path() + args + ']' - else: - desc = tg.get_description() - if desc is None: - desc = '' - else: - desc = ' [' + desc + ']' - detailed_desc = tg.__class__.__name__ + desc + desc = ' [' + tg.get_description() + ']' + detailed_desc = tg.__class__.__name__ + desc return detailed_desc @@ -1062,7 +1161,7 @@ def show_targets(self): print(colorize(FontStyle.BOLD + '\n-=[ Available Targets ]=-\n', rgb=Color.INFO)) idx = 0 for tg in self.get_available_targets(): - name = self._get_detailed_target_desc(tg) + name = self.available_targets_desc[tg] msg = "[{:d}] {:s}".format(idx, name) @@ -1070,14 +1169,15 @@ def show_targets(self): if probes: msg += '\n \-- monitored by:' for p in probes: - pname, delay = self._extract_info_from_probe(p) + pobj, delay = self._extract_info_from_probe(p) + pname = pobj.__name__ if delay: msg += " {:s}(refresh={:.2f}s),".format(pname, delay) else: msg += " {:s},".format(pname) msg = msg[:-1] - if self.__current_tg == idx: + if idx in self._tg_ids: msg = colorize(FontStyle.BOLD + msg, rgb=Color.SELECTED) else: msg = colorize(msg, rgb=Color.SUBINFO) @@ -1092,21 +1192,38 @@ def dynamic_generator_ids(self): @EnforceOrder(accepted_states=['S2']) def show_fmk_internals(self): - if not self.tg.supported_feedback_mode: - fbk_mode = 'Target does not provide feedback' - elif self.tg.fbk_wait_full_time_slot_mode: - fbk_mode = self.tg.fbk_wait_full_time_slot_msg - else: - fbk_mode = self.tg.fbk_wait_until_recv_msg print(colorize(FontStyle.BOLD + '\n-=[ FMK Internals ]=-\n', rgb=Color.INFO)) + print(colorize(' [ General Information ]', rgb=Color.INFO)) + print(colorize(' FmkDB enabled: ', rgb=Color.SUBINFO) + repr(self.fmkDB.enabled)) + print(colorize(' Workspace enabled: ', rgb=Color.SUBINFO) + repr(self._wkspace_enabled)) print(colorize(' Fuzz delay: ', rgb=Color.SUBINFO) + str(self._delay)) print(colorize(' Number of data sent in burst: ', rgb=Color.SUBINFO) + str(self._burst)) - print(colorize(' Target health-check timeout: ', rgb=Color.SUBINFO) + str(self._hc_timeout)) - print(colorize(' Target feedback timeout: ', rgb=Color.SUBINFO) + str(self.tg.feedback_timeout)) - print(colorize(' Target feedback mode: ', rgb=Color.SUBINFO) + fbk_mode) - print(colorize(' Workspace enabled: ', rgb=Color.SUBINFO) + repr(self._wkspace_enabled)) - print(colorize(' FmkDB enabled: ', rgb=Color.SUBINFO) + repr(self.fmkDB.enabled)) + print(colorize(' Target(s) health-check timeout: ', rgb=Color.SUBINFO) + str(self._hc_timeout_max)) + + for tg_id, tg in self.targets.items(): + if not tg.supported_feedback_mode: + fbk_mode = 'Target does not provide feedback' + elif tg.fbk_wait_full_time_slot_mode: + fbk_mode = tg.fbk_wait_full_time_slot_msg + else: + fbk_mode = tg.fbk_wait_until_recv_msg + fbk_timeout = str(tg.feedback_timeout) + tg_name = self.available_targets_desc[tg] + + print(colorize('\n [ Target Specific Information - ({:d}) {!s} ]'.format(tg_id, tg_name), rgb=Color.INFO)) + print(colorize(' Feedback timeout: ', rgb=Color.SUBINFO) + fbk_timeout) + print(colorize(' Feedback mode: ', rgb=Color.SUBINFO) + fbk_mode) + + + @EnforceOrder(accepted_states=['S2']) + def show_knowledge(self): + k = self.prj.knowledge_source + print(colorize(FontStyle.BOLD + '\n-=[ Status of Knowledge ]=-\n', rgb=Color.INFO)) + if k: + print(colorize(str(k), rgb=Color.SUBINFO)) + else: + print(colorize('No knowledge', rgb=Color.SUBINFO)) @EnforceOrder(accepted_states=['20_load_prj','25_load_dm','S1','S2']) def projects(self): @@ -1140,24 +1257,35 @@ def show_data_models(self): print(colorize(FontStyle.BOLD + '\n-=[ Data Models ]=-\n', rgb=Color.INFO)) idx = 0 for dm in self.__iter_data_models(): - print(colorize('[%d] ' % idx + dm.name, rgb=Color.SUBINFO)) + if dm is self.dm: + print(colorize(FontStyle.BOLD + '[{:d}] {!s}'.format(idx, dm.name), rgb=Color.SELECTED)) + else: + print(colorize('[{:d}] {!s}'.format(idx, dm.name), rgb=Color.SUBINFO)) idx += 1 - def __init_fmk_internals_step1(self, prj, dm): + def _init_fmk_internals_step1(self, prj, dm): self.prj = prj self.dm = dm + # self.dm.knowledge_source = prj.knowledge_source self.lg = self.__logger_dict[prj] + + self.targets = {} try: - self.tg = self.__target_dict[prj][self.__current_tg] + for tg_id in self._tg_ids: + self.targets[tg_id] = self.__target_dict[prj][tg_id] + # self.targets[tg_id].tg_id = tg_id except IndexError: - self.__current_tg = 0 - self.tg = self.__target_dict[prj][self.__current_tg] + self.set_error(msg="Invalid Target ID(s). Enable the EmptyTarget (0) only.", code=Error.FmkWarning) + self.targets = {0: self.__target_dict[prj][0]} + self._tg_ids = [0] + + self._update_targets_desc(prj) - self.tg_name = self._get_detailed_target_desc(self.tg) + for tg in self.targets.values(): + tg.set_logger(self.lg) + tg.set_data_model(self.dm) - self.tg.set_logger(self.lg) - self.tg.set_data_model(self.dm) - self.prj.set_target(self.tg) + self.prj.set_targets(self.targets) self.prj.set_data_model(self.dm) if self.__first_loading: @@ -1170,14 +1298,16 @@ def __init_fmk_internals_step1(self, prj, dm): self._tactics.clear_disruptor_clones() self._tactics = self.__st_dict[dm] + if self.prj.project_scenarios: + self._tactics.register_scenarios(*self.prj.project_scenarios) self.mon = self.__monitor_dict[prj] - self.mon.set_target(self.tg) + self.mon.set_targets(self.targets) self.mon.set_logger(self.lg) self.mon.set_data_model(self.dm) self.__initialized_dmakers = self.__initialized_dmaker_dict[prj] - def __init_fmk_internals_step2(self, prj, dm): + def _init_fmk_internals_step2(self, prj, dm): self._recompute_current_generators() # need the logger active self.__reset_fmk_internals() @@ -1211,15 +1341,15 @@ def load_data_model(self, dm=None, name=None): if dm not in self.dm_list: return False - if self.__is_started(): + if self._is_started(): self.cleanup_all_dmakers() self.dm = dm self.prj.set_data_model(self.dm) - if hasattr(self, 'tg'): - self.tg.set_data_model(self.dm) - if hasattr(self, 'mon'): + for tg in self.targets.values(): + tg.set_data_model(self.dm) + if self.mon: self.mon.set_data_model(self.dm) - if self.__is_started(): + if self._is_started(): self._cleanup_dm_attrs_from_fmk() ok = self._load_data_model() if not ok: @@ -1244,7 +1374,7 @@ def load_multiple_data_model(self, dm_list=None, name_list=None, reload_dm=False if dm not in self.dm_list: return False - if self.__is_started(): + if self._is_started(): self.cleanup_all_dmakers() new_dm = DataModel() @@ -1253,7 +1383,7 @@ def load_multiple_data_model(self, dm_list=None, name_list=None, reload_dm=False name = '' for dm in dm_list: name += dm.name + '+' - if not reload_dm: + if not reload_dm or not self._is_started(): self.dm = dm self._cleanup_dm_attrs_from_fmk() ok = self._load_data_model() @@ -1280,7 +1410,7 @@ def load_multiple_data_model(self, dm_list=None, name_list=None, reload_dm=False if reload_dm or not is_dm_name_exists: self.fmkDB.insert_data_model(new_dm.name) self.__add_data_model(new_dm, new_tactics, - (None, dm_list), + [None, dm_list], reload_dm=reload_dm) # In this case DynGens have already been generated through @@ -1296,11 +1426,11 @@ def load_multiple_data_model(self, dm_list=None, name_list=None, reload_dm=False self.dm = new_dm self.prj.set_data_model(self.dm) - if hasattr(self, 'tg'): - self.tg.set_data_model(self.dm) - if hasattr(self, 'mon'): + for tg in self.targets.values(): + tg.set_data_model(self.dm) + if self.mon: self.mon.set_data_model(self.dm) - if self.__is_started(): + if self._is_started(): self._cleanup_dm_attrs_from_fmk() ok = self._load_data_model() if not ok: @@ -1308,6 +1438,10 @@ def load_multiple_data_model(self, dm_list=None, name_list=None, reload_dm=False return True + def _update_targets_desc(self, prj): + self.available_targets_desc = {} + for tg in self.__target_dict[prj]: + self.available_targets_desc[tg] = self._get_detailed_target_desc(tg) @EnforceOrder(accepted_states=['20_load_prj','25_load_dm','S1','S2']) def get_project_by_name(self, name): @@ -1321,7 +1455,7 @@ def get_project_by_name(self, name): @EnforceOrder(accepted_states=['20_load_prj','25_load_dm','S1','S2'], final_state='S2') - def run_project(self, prj=None, name=None, tg=None, dm_name=None): + def run_project(self, prj=None, name=None, tg_ids=None, dm_name=None): ok = self.load_project(prj=prj, name=name) if not ok: return False @@ -1333,21 +1467,30 @@ def run_project(self, prj=None, name=None, tg=None, dm_name=None): else: dm_name = self.prj.default_dm - if isinstance(dm_name, list): - ok = self.load_multiple_data_model(name_list=dm_name) + if self._dm_to_be_reloaded: + self._reload_dm(dm_name=dm_name) + ok = self.is_ok() else: - ok = self.load_data_model(name=dm_name) + if isinstance(dm_name, list): + ok = self.load_multiple_data_model(name_list=dm_name) + else: + ok = self.load_data_model(name=dm_name) if not ok: + self._dm_to_be_reloaded = True return False + else: + self._dm_to_be_reloaded = False - if tg is not None: - assert(isinstance(tg, int)) - self.__set_target(tg) + if tg_ids is not None: + if isinstance(tg_ids, int): + self._load_targets([tg_ids]) + else: + self._load_targets(tg_ids) else: - self.__set_target(0) + self._load_targets([0]) - return self.launch() + return self._launch() @EnforceOrder(accepted_states=['20_load_prj','25_load_dm','S1','S2'], final_state='25_load_dm') @@ -1361,38 +1504,43 @@ def load_project(self, prj=None, name=None): if prj not in self.prj_list: return False + self._stop_fmk_plumbing() self.prj = prj - self.__stop_fmk_plumbing() + self._update_targets_desc(prj) return True @EnforceOrder(accepted_states=['S1'], final_state='S2') def launch(self): - if not self.__prj_to_be_reloaded: - self.__init_fmk_internals_step1(self.prj, self.dm) - self.__start_fmk_plumbing() - if self.is_not_ok(): - self.__stop_fmk_plumbing() - return False - - self.__init_fmk_internals_step2(self.prj, self.dm) - return True - + if not self._dm_to_be_reloaded: + return self._launch() else: - self.__prj_to_be_reloaded = False - self.__reload_all() + self._dm_to_be_reloaded = False + self._reload_dm() + self._launch() + # self._reload_all() return True + def _launch(self): + self._init_fmk_internals_step1(self.prj, self.dm) + self._start_fmk_plumbing() + if self.is_not_ok(): + self._stop_fmk_plumbing() + return False + + self._init_fmk_internals_step2(self.prj, self.dm) + return True + def is_target_enabled(self): return self.__tg_enabled - def __enable_target(self): + def _enable_target(self): self.__tg_enabled = True self.mon.enable_hooks() - def __disable_target(self): + def _disable_target(self): self.__tg_enabled = False self.mon.disable_hooks() @@ -1427,52 +1575,99 @@ def set_fuzz_burst(self, val, do_record=False): return False @EnforceOrder(accepted_states=['S1','S2']) - def set_health_check_timeout(self, timeout, do_record=False, do_show=True): + def set_health_check_timeout(self, timeout, target=None, do_record=False, do_show=True): if timeout >= 0: - self._hc_timeout = timeout + if target is None: + self._hc_timeout = {} + for tg in self.targets.values(): + self._hc_timeout[tg] = timeout + else: + self._hc_timeout[target] = timeout + self._hc_timeout_max = max(self._hc_timeout.values()) if do_show or do_record: - self.lg.log_fmk_info('Target health-check timeout = {:.1f}s'.format(self._hc_timeout), - do_record=do_record) + if target is None: + self.lg.log_fmk_info('Target(s) health-check timeout = {:.1f}s'.format(timeout), + do_record=do_record) + else: + tg_desc = self._get_detailed_target_desc(target) + self.lg.log_fmk_info('Target {!s} health-check timeout = {:.1f}s'.format(tg_desc, timeout), + do_record=do_record) + return True else: self.lg.log_fmk_info('Wrong timeout value!', do_record=False) return False @EnforceOrder(accepted_states=['S1','S2']) - def set_feedback_timeout(self, timeout, do_record=False, do_show=True): + def set_feedback_timeout(self, timeout, tg_id=None, do_record=False, do_show=True): + + if tg_id is None: + max_sending_delay = 0 + for tg in self.targets.values(): + max_sending_delay = max(max_sending_delay, tg.sending_delay) + if timeout is None: # This case occurs in self._do_sending_and_logging_init() # if the Target has not defined a feedback_timeout (like the EmptyTarget) - self._recompute_health_check_timeout(timeout, self.tg.sending_delay, do_show=do_show) + if tg_id is None: + self._recompute_health_check_timeout(timeout, max_sending_delay, do_show=do_show) + else: + tg = self.targets[tg_id] + self._recompute_health_check_timeout(timeout, tg.sending_delay, target=tg, do_show=do_show) + elif timeout >= 0: - self.tg.set_feedback_timeout(timeout) - if do_show or do_record: - self.lg.log_fmk_info('Target feedback timeout = {:.1f}s'.format(timeout), - do_record=do_record) - self._recompute_health_check_timeout(timeout, self.tg.sending_delay, do_show=do_show) + if tg_id is None: + for tg in self.targets.values(): + tg.set_feedback_timeout(timeout) + self._recompute_health_check_timeout(timeout, max_sending_delay, do_show=do_show) + if do_show or do_record: + self.lg.log_fmk_info('Target(s) feedback timeout = {:.1f}s'.format(timeout), + do_record=do_record) + else: + tg = self.targets[tg_id] + tg.set_feedback_timeout(timeout) + self._recompute_health_check_timeout(timeout, tg.sending_delay, target=tg, do_show=do_show) + if do_show or do_record: + tg_desc = self._get_detailed_target_desc(tg) + self.lg.log_fmk_info('Target {!s} feedback timeout = {:.1f}s'.format(tg_desc, timeout), + do_record=do_record) return True else: self.lg.log_fmk_info('Wrong timeout value!', do_record=False) return False @EnforceOrder(accepted_states=['S1','S2']) - def set_feedback_mode(self, mode, do_record=False, do_show=True): - ok = self.tg.set_feedback_mode(mode) - if not ok: - self.set_error('The target does not support this feedback Mode', code=Error.CommandError) - elif do_show or do_record: - if self.tg.fbk_wait_full_time_slot_mode: - msg = 'Feedback Mode = ' + self.tg.fbk_wait_full_time_slot_msg - else: - msg = 'Feedback Mode = ' + self.tg.fbk_wait_until_recv_msg - self.lg.log_fmk_info(msg, do_record=do_record) + def set_feedback_mode(self, mode, tg_id=None, do_record=False, do_show=True): + + def _set_fbk_mode(tg): + ok = tg.set_feedback_mode(mode) + if not ok: + self.set_error('The target does not support this feedback Mode', code=Error.CommandError) + elif do_show or do_record: + if tg.fbk_wait_full_time_slot_mode: + msg = 'Feedback Mode = ' + tg.fbk_wait_full_time_slot_msg + else: + msg = 'Feedback Mode = ' + tg.fbk_wait_until_recv_msg + self.lg.log_fmk_info(msg, do_record=do_record) + + if tg_id is None: + for tg in self.targets.values(): + _set_fbk_mode(tg) + else: + tg = self.targets[tg_id] + _set_fbk_mode(tg) @EnforceOrder(accepted_states=['S1','S2']) - def switch_feedback_mode(self, do_record=False, do_show=True): - if self.tg.fbk_wait_full_time_slot_mode: - self.set_feedback_mode(Target.FBK_WAIT_UNTIL_RECV, do_record=do_record, do_show=do_show) + def switch_feedback_mode(self, tg_id, do_record=False, do_show=True): + if tg_id not in self.targets: + self.set_error('The selected target is not enabled', code=Error.CommandError) + return + + tg = self.targets[tg_id] + if tg.fbk_wait_full_time_slot_mode: + self.set_feedback_mode(Target.FBK_WAIT_UNTIL_RECV, tg_id=tg_id, do_record=do_record, do_show=do_show) else: - self.set_feedback_mode(Target.FBK_WAIT_FULL_TIME, do_record=do_record, do_show=do_show) + self.set_feedback_mode(Target.FBK_WAIT_FULL_TIME, tg_id=tg_id, do_record=do_record, do_show=do_show) # Used to introduce some delay after sending data def __delay_fuzzing(self): @@ -1542,66 +1737,99 @@ def _do_before_sending_data(self, data_list): def _do_after_sending_data(self, data_list): self._handle_data_callbacks(data_list, hook=HOOK.after_sending) + self.prj.notify_data_sending(data_list, self._current_sent_date, self.targets) def _do_sending_and_logging_init(self, data_list): - - # If feedback_timeout = 0 then we don't consider residual feedback. - # We try to avoid unnecessary latency in this case, as well as - # to avoid retrieving some feedback that could be a trigger for sending the next data - # (e.g., with a NetworkTarget in server_mode + wait_for_client) - do_residual_fbk_gathering = True if self.tg.feedback_timeout is None else self.tg.feedback_timeout > 0 - for d in data_list: + mapping = self.prj.scenario_target_mapping.get(d.scenario_dependence, None) + if d.feedback_timeout is not None: - self.set_feedback_timeout(d.feedback_timeout) + tg_ids = self._vtg_to_tg(d) + for tg_id in tg_ids: + self.set_feedback_timeout(d.feedback_timeout, tg_id=tg_id) + if d.feedback_mode is not None: - self.set_feedback_mode(d.feedback_mode) + tg_ids = self._vtg_to_tg(d) + for tg_id in tg_ids: + self.set_feedback_mode(d.feedback_mode, tg_id=tg_id) blocked_data = list(filter(lambda x: x.is_blocked(), data_list)) data_list = list(filter(lambda x: not x.is_blocked(), data_list)) - user_interrupt = False + user_interrupt, go_on = self._collect_residual_feedback(cond1=(self._burst_countdown == self._burst), + cond2=(not blocked_data)) + + if blocked_data: + self._handle_data_callbacks(blocked_data, hook=HOOK.after_fbk) + self.fmkDB.flush_current_feedback() + + if user_interrupt: + raise UserInterruption + elif go_on: + return data_list + else: + raise TargetFeedbackError + + def collect_residual_feedback(self): + if self._collect_residual_feedback(True, True)[0]: + raise UserInterruption + + def _collect_residual_feedback(self, cond1, cond2): + # If feedback_timeout = 0 then we don't consider residual feedback. + # We try to avoid unnecessary latency in this case, as well as + # to avoid retrieving some feedback that could be a trigger for sending the next data + # (e.g., with a NetworkTarget in server_mode + wait_for_client) + targets_to_retrieve_fbk = {} + do_residual_fbk_gathering = False + for tg_id, tg in self.targets.items(): + cond = True if tg.feedback_timeout is None else tg.feedback_timeout > 0 + if cond: + do_residual_fbk_gathering = True + targets_to_retrieve_fbk[tg_id] = tg + go_on = True - if self._burst_countdown == self._burst and do_residual_fbk_gathering: + fbk_timeout = {} + user_interrupt = False + if cond1 and do_residual_fbk_gathering: # log residual just before sending new data to avoid # polluting feedback logs of the next emission - if not blocked_data: - fbk_timeout = self.tg.feedback_timeout - # we change feedback timeout as the target could use it to determine if it is + if cond2: + for tg in targets_to_retrieve_fbk.values(): + fbk_timeout[tg] = tg.feedback_timeout + # we change feedback timeout as the targets could use it to determine if they are # ready to accept new data (check_target_readiness). For instance, the NetworkTarget # launch a thread when collect_feedback_without_sending() is called for a duration # of 'feedback_timeout'. self.set_feedback_timeout(0, do_show=False) - # print('\nDBG: before collecting residual', self.tg._feedback_handled) - if self.tg.collect_feedback_without_sending(): - # We have to make sure the target is ready for sending data after + collected = False + for tg in targets_to_retrieve_fbk.values(): + if tg.collect_feedback_without_sending(): + collected = True + + if collected: + # We have to make sure the targets are ready for sending data after # collecting feedback. - # print('\nDBG: collecting residual', self.tg._feedback_handled) ret = self.check_target_readiness() - # print('\nDBG: target_ready', self.tg._feedback_handled) user_interrupt = ret == -2 + go_on = self.log_target_residual_feedback() - # print('\nDBG: residual fbk logged') - if not blocked_data: - self.set_feedback_timeout(fbk_timeout, do_show=False) + if cond2: + for tg_id, tg in targets_to_retrieve_fbk.items(): + self.set_feedback_timeout(fbk_timeout[tg], tg_id=tg_id, do_show=False) - self.tg.cleanup() - self.monitor_probes(prefix='Probe Status Before Sending Data') + for tg in targets_to_retrieve_fbk.values(): + tg.cleanup() - if blocked_data: - self._handle_data_callbacks(blocked_data, hook=HOOK.after_fbk) + self.monitor_probes(prefix='Probe Status Before Sending Data') + + return user_interrupt, go_on - if user_interrupt: - raise UserInterruption - elif go_on: - return data_list - else: - raise TargetFeedbackError def _do_after_feedback_retrieval(self, data_list): self._handle_data_callbacks(data_list, hook=HOOK.after_fbk) + self.fmkDB.flush_current_feedback() def _do_after_dmaker_data_retrieval(self, data): self._handle_data_callbacks([data], hook=HOOK.after_dmaker_production) @@ -1655,9 +1883,10 @@ def _handle_data_desc(self, data_desc, resolve_dataprocess=True, original_data=N if data is None: self.set_error(msg='Data creation process has yielded!', code=Error.DPHandOver) - print('\n+++ DP yield', data_desc.auto_regen) return None + data.tg_ids = data_desc.vtg_ids + elif isinstance(data_desc, str): try: node = self.dm.get_atom(data_desc) @@ -1668,6 +1897,8 @@ def _handle_data_desc(self, data_desc, resolve_dataprocess=True, original_data=N else: data = Data(node) data.generate_info_from_content(original_data=original_data) + if original_data is not None: + data.tg_ids = original_data.tg_ids else: self.set_error( msg='Data descriptor type is not recognized {!s}!'.format(type(data_desc)), @@ -1682,10 +1913,15 @@ def _handle_data_desc(self, data_desc, resolve_dataprocess=True, original_data=N def _handle_data_callbacks(self, data_list, hook, resolve_dataprocess=True): new_data_list = [] + stop_data_list_processing = False + for data in data_list: + if stop_data_list_processing: + break + try: if hook == HOOK.after_fbk: - data.run_callbacks(feedback=self.feedback_handler, hook=hook) + data.run_callbacks(feedback=self.feedback_gate, hook=hook) else: data.run_callbacks(feedback=None, hook=hook) except: @@ -1695,20 +1931,36 @@ def _handle_data_callbacks(self, data_list, hook, resolve_dataprocess=True): continue new_data = data + data_tg_ids = data.tg_ids if data.tg_ids is not None else [self._tg_ids[0]] pending_ops = data.pending_callback_ops(hook=hook) if pending_ops: for op in pending_ops: + # CallBackOps.Set_FbkTimeout is obsolete. Not used by scenario, only used in + # tuto_strategy.py as an example. Note that fbk timeout is dealt directly at Data() + # level by self._do_sending_and_logging_init() fbk_timeout = op[CallBackOps.Set_FbkTimeout] if fbk_timeout is not None: self.set_feedback_timeout(fbk_timeout) - data_desc = op[CallBackOps.Replace_Data] - if data_desc is not None: + returned_obj = op[CallBackOps.Replace_Data] + if returned_obj is not None: + # This means that data_list will be replaced by something else, thus ignore + # current data_list and start from scratch. + # In case of Scenario handling with a multi data step, we skip the remaining + # data in the list because they will be regenerated or new ones will prevail. + # Indeed, the Replace_Data callback is triggered twice: + # in Hook.before_sending_step1 and in Hook.before_sending_step2. + stop_data_list_processing = True + + data_desc, vtg_ids_list = returned_obj + if vtg_ids_list is None: + vtg_ids_list = itertools.repeat(None) + new_data = [] first_step = True - for d_desc in data_desc: + for d_desc, vtg_ids in zip(data_desc, vtg_ids_list): data_tmp = self._handle_data_desc(d_desc, resolve_dataprocess=resolve_dataprocess, original_data=data) @@ -1716,9 +1968,17 @@ def _handle_data_callbacks(self, data_list, hook, resolve_dataprocess=True): if first_step: first_step = False data_tmp.copy_callback_from(data) + data_tmp.tg_ids = vtg_ids + data_tmp.scenario_dependence = data.scenario_dependence new_data.append(data_tmp) else: + # We mark the data unusable in order to make sending methods + # aware of specific events that should stop the sending process. + # In this case it is either the normal end of a scenario or an error + # within a scenario step. newd = Data() + newd.tg_ids = vtg_ids + newd.scenario_dependence = data.scenario_dependence newd.make_unusable() new_data = [newd] break @@ -1726,18 +1986,20 @@ def _handle_data_callbacks(self, data_list, hook, resolve_dataprocess=True): for idx in op[CallBackOps.Del_PeriodicData]: self._unregister_task(idx) + final_data_tg_ids = self._vtg_to_tg(data) for idx, obj in op[CallBackOps.Add_PeriodicData].items(): data_desc, period = obj if isinstance(data_desc, DataProcess): # In this case each time we send the periodic we walk through the process # (thus, sending a new data each time) periodic_data = data_desc - func = self._send_periodic + func = functools.partial(self._send_periodic, final_data_tg_ids) else: periodic_data = self._handle_data_desc(data_desc, resolve_dataprocess=resolve_dataprocess, original_data=data) - func = self.tg.send_data_sync + targets = [self.targets[x] for x in final_data_tg_ids] + func = [tg.send_data_sync for tg in targets] if periodic_data is not None: task = FmkTask(idx, func, periodic_data, period=period, @@ -1752,17 +2014,42 @@ def _handle_data_callbacks(self, data_list, hook, resolve_dataprocess=True): if isinstance(new_data, list): for newd in new_data: - if not newd.is_unusable(): - new_data_list.append(newd) - elif not new_data.is_unusable(): + new_data_list.append(newd) + else: new_data_list.append(new_data) return new_data_list - def _send_periodic(self, data_desc): + def _vtg_to_tg(self, data): + mapping = self.prj.scenario_target_mapping.get(data.scenario_dependence, None) + if data.tg_ids is None: + tg_ids = mapping.get(None, self._tg_ids[0]) if mapping else self._tg_ids[0] + tg_ids = [tg_ids] + else: + if mapping: + tg_ids = [mapping.get(tg_id, tg_id) for tg_id in data.tg_ids] + else: + tg_ids = data.tg_ids + + valid_tg_ids = [] + for i in tg_ids: + if i not in self._tg_ids: + self.set_error("WARNING: An access attempt occurs on a disabled target: '({:d}) {!s}' " + "It will be redirected to the first enabled target." + .format(i, self.get_available_targets()[i]), + code=Error.FmkWarning) + i = self._tg_ids[0] + if self._debug_mode: + raise ValueError('Access attempt occurs on a disabled target') + valid_tg_ids.append(i) + + return valid_tg_ids + + def _send_periodic(self, tg_ids, data_desc): data = self._handle_data_desc(data_desc) if data is not None: - self.tg.send_data_sync(data) + for tg in [self.targets[tg_id] for tg_id in tg_ids]: + tg.send_data_sync(data) else: self.set_error(msg="Data descriptor handling returned 'None'!", code=Error.UserCodeError) raise DataProcessTermination @@ -1776,7 +2063,7 @@ def _unregister_task(self, id, ign_error=False): '(Task ID #{!s})'.format(id)) elif not ign_error: self.set_error('ERROR: Task ID #{!s} does not exist. ' - 'Cannot unregister.'.format(id, code=Error.UserCodeError)) + 'Cannot unregister.'.format(id), code=Error.UserCodeError) def _register_task(self, id, task): with self._task_list_lock: @@ -1785,7 +2072,7 @@ def _register_task(self, id, task): task.start() else: self.set_error('WARNING: Task ID #{!s} already exists. ' - 'Task ignored.'.format(id, code=Error.UserCodeError)) + 'Task ignored.'.format(id), code=Error.UserCodeError) def _cleanup_tasks(self): for id in self._task_list: @@ -1818,16 +2105,17 @@ def send_data_and_log(self, data_list, original_data=None, verbose=False): if not data_list: return True - data_list = self._send_data(data_list, add_preamble=True) + data_list = self._send_data(data_list) + + if self._sending_error or self._stop_sending: + return False + if data_list is None: # In this case, some data callbacks have triggered to block the emission of # what was in data_list. We go on because this is a normal behavior (especially in the # context of Scenario() execution). return True - if self._sending_error: - return False - # All feedback entries that are available for relevant framework users (scenario # callbacks, operators, ...) are flushed just after sending a new data because it # means the previous feedback entries are obsolete. @@ -1854,6 +2142,10 @@ def send_data_and_log(self, data_list, original_data=None, verbose=False): for dt in data_list: dt.make_recordable() + # When checking target readiness, feedback timeout is taken into account indirectly + # through the call to Target.is_target_ready_for_new_data() + cont0 = self.check_target_readiness() >= 0 + if multiple_data: self._log_data(data_list, original_data=original_data, verbose=verbose) @@ -1861,16 +2153,6 @@ def send_data_and_log(self, data_list, original_data=None, verbose=False): orig = original_data[0] if orig_data_provided else None self._log_data(data_list[0], original_data=orig, verbose=verbose) - # When checking target readiness, feedback timeout is taken into account indirectly - # through the call to Target.is_target_ready_for_new_data() - cont0 = self.check_target_readiness() >= 0 - - ack_date = self.tg.get_last_target_ack_date() - self.lg.log_target_ack_date(ack_date) - - if cont0: - cont0 = self.__delay_fuzzing() - cont1 = True cont2 = True # That means this is the end of a burst @@ -1883,15 +2165,19 @@ def send_data_and_log(self, data_list, original_data=None, verbose=False): if self._burst_countdown == self._burst: # We handle probe feedback if any cont2 = self.monitor_probes(force_record=True) - self.tg.cleanup() + for tg in self._currently_used_targets: + tg.cleanup() self._do_after_feedback_retrieval(data_list) + if cont0: + cont0 = self.__delay_fuzzing() + return cont0 and cont1 and cont2 @EnforceOrder(accepted_states=['S2']) - def _send_data(self, data_list, add_preamble=False): + def _send_data(self, data_list): ''' @data_list: either a list of Data() or a Data() ''' @@ -1918,29 +2204,52 @@ def _send_data(self, data_list, add_preamble=False): self.mon.notify_error() return None + self._stop_sending = False + if data_list[0].is_unusable(): + self.set_error("_send_data(): A DataProcess has yielded. No more data to send.", + code=Error.NoMoreData) + self.mon.notify_error() + self._stop_sending = True + return None + + self._setup_new_sending() self._sending_error = False - try: - if len(data_list) == 1: - self.tg.send_data_sync(data_list[0], from_fmk=True) - elif len(data_list) > 1: - self.tg.send_multiple_data_sync(data_list, from_fmk=True) + + used_targets = [] + for d in data_list: + tg_ids = self._vtg_to_tg(d) + for tg_id in tg_ids: + if tg_id not in self.targets: + self.mon.notify_error() + self.set_error("_send_data(): Invalid Target ID ({:d})".format(tg_id), code=Error.FmkError) + self._sending_error = True + return None + + tg = self.targets[tg_id] + tg.add_pending_data(d) + used_targets.append(tg) + + seen = set() + self._currently_used_targets = [x for x in used_targets if not (x in seen or seen.add(x))] + + for tg in self._currently_used_targets: + try: + tg.send_pending_data(from_fmk=True) + except TargetStuck as e: + self.lg.log_target_feedback_from( + source=FeedbackSource(self), + content='*** WARNING: Unable to send data to the target! [reason: {!s}]'.format(e), + status_code=-1, + timestamp=datetime.datetime.now(), + ) + self.mon.notify_error() + self._sending_error = True + except: + self._handle_user_code_exception() + self.mon.notify_error() + self._sending_error = True else: - raise ValueError - except TargetStuck as e: - self.lg.log_target_feedback_from( - '*** WARNING: Unable to send data to the target! [reason: {!s}]'.format(e), - datetime.datetime.now(), status_code=-1, source='Fuddly FmK' - ) - self.mon.notify_error() - self._sending_error = True - except: - self._handle_user_code_exception() - self.mon.notify_error() - self._sending_error = True - else: - if add_preamble: - self.new_transfer_preamble() - self.mon.notify_data_sending_event() + self.mon.notify_data_sending_event() self._do_after_sending_data(data_list) @@ -1959,6 +2268,7 @@ def _log_data(self, data_list, original_data=None, verbose=False): return self.group_id += 1 + self._recovered_tgs = None gen = self.__current_gen if original_data is None: @@ -1979,7 +2289,7 @@ def _log_data(self, data_list, original_data=None, verbose=False): if multiple_data: self.lg.log_fmk_info("MULTIPLE DATA EMISSION", nl_after=True, delay_recording=True) - for idx, dt in zip(range(len(data_list)), data_list): + for idx, dt in enumerate(data_list): dt_mk_h = dt.get_history() if multiple_data: self.lg.log_fmk_info("Data #%d" % (idx+1), nl_before=True, delay_recording=True) @@ -2055,9 +2365,14 @@ def _log_data(self, data_list, original_data=None, verbose=False): self.lg.log_data(dt, verbose=verbose) + tg_ids = self._vtg_to_tg(dt) + for tg_id in tg_ids: + tg = self.targets[tg_id] + ack_date = tg.get_last_target_ack_date() + self.lg.set_target_ack_date(FeedbackSource(tg), date=ack_date) if self.fmkDB.enabled: - data_id = self.lg.commit_log_entry(self.group_id, self.prj.name, self.tg_name) + data_id = self.lg.commit_data_table_entry(self.group_id, self.prj.name) if data_id is None: self.lg.print_console('### Data not recorded in FmkDB', rgb=Color.DATAINFO, nl_after=True) @@ -2068,59 +2383,71 @@ def _log_data(self, data_list, original_data=None, verbose=False): if multiple_data: self.lg.log_fn("--------------------------", rgb=Color.SUBINFO) + self.lg.log_target_ack_date() + + self.lg.reset_current_state() @EnforceOrder(accepted_states=['S2']) - def new_transfer_preamble(self): + def _setup_new_sending(self): if self._burst > 1 and self._burst_countdown == self._burst: p = "\n::[ START BURST ]::\n" else: p = "\n" - self.lg.start_new_log_entry(preamble=p) + self._current_sent_date = self.lg.start_new_log_entry(preamble=p) @EnforceOrder(accepted_states=['S2']) def log_target_feedback(self): - err_detected1, err_detected2 = False, False + collected_err, err_detected2 = None, False + ok = True if self.__tg_enabled: if self._burst > 1: p = "::[ END BURST ]::\n" else: p = None try: - err_detected1 = self.lg.log_collected_target_feedback(preamble=p) + collected_err = self.lg.log_collected_feedback(preamble=p) except NotImplementedError: pass - finally: - err_detected2 = self._log_directly_retrieved_target_feedback(preamble=p) - go_on = self._recover_target() if err_detected1 or err_detected2 else True + for tg in self.targets.values(): + err_detected1 = collected_err.get(tg, False) if collected_err else False + err_detected2 = self._log_directly_retrieved_target_feedback(tg=tg, preamble=p) + go_on = self._recover_target(tg) if err_detected1 or err_detected2 else True + if not go_on: + ok = False - return go_on + return ok @EnforceOrder(accepted_states=['S2']) def log_target_residual_feedback(self): - err_detected1, err_detected2 = False, False + collected_err, err_detected2 = None, False + ok = True if self.__tg_enabled: p = "*** RESIDUAL TARGET FEEDBACK ***" e = "********************************" try: - err_detected1 = self.lg.log_collected_target_feedback(preamble=p, epilogue=e) + collected_err = self.lg.log_collected_feedback(preamble=p, epilogue=e) except NotImplementedError: pass - finally: - err_detected2 = self._log_directly_retrieved_target_feedback(preamble=p, epilogue=e) - go_on = self._recover_target() if err_detected1 or err_detected2 else True + for tg in self.targets.values(): + err_detected1 = collected_err.get(tg, False) if collected_err else False + err_detected2 = self._log_directly_retrieved_target_feedback(tg=tg, + preamble=p, epilogue=e) + go_on = self._recover_target(tg) if err_detected1 or err_detected2 else True + if not go_on: + ok = False - return go_on + return ok - def _log_directly_retrieved_target_feedback(self, preamble=None, epilogue=None): + def _log_directly_retrieved_target_feedback(self, tg, preamble=None, epilogue=None): """ This method is to be used when the target does not make use - of Logger.collect_target_feedback() facility. We thus try to + of Logger.collect_feedback() facility. We thus try to access the feedback from Target directly """ err_detected = False - tg_fbk = self.tg.get_feedback() + tg_fbk = tg.get_feedback() if tg_fbk is not None: err_code = tg_fbk.get_error_code() if err_code is not None and err_code < 0: @@ -2130,15 +2457,19 @@ def _log_directly_retrieved_target_feedback(self, preamble=None, epilogue=None): for ref, fbk, status, tstamp in tg_fbk.iter_and_cleanup_collector(): if status < 0: err_detected = True - self.lg.log_target_feedback_from(fbk, tstamp, preamble=preamble, - epilogue=epilogue, - source=ref, status_code=status) + self.lg.log_target_feedback_from(source=FeedbackSource(tg, subref=ref), + content=fbk, + status_code=status, + timestamp=tstamp, + preamble=preamble, + epilogue=epilogue) raw_fbk = tg_fbk.get_bytes() if raw_fbk is not None: - self.lg.log_target_feedback_from(raw_fbk, - tg_fbk.get_timestamp(), + self.lg.log_target_feedback_from(source=FeedbackSource(tg), + content=raw_fbk, status_code=err_code, + timestamp=tg_fbk.get_timestamp(), preamble=preamble, epilogue=epilogue) @@ -2152,32 +2483,40 @@ def check_target_readiness(self): if self.__tg_enabled: t0 = datetime.datetime.now() + signal.signal(signal.SIGINT, sig_int_handler) + ret = 0 + tg = None + # Wait until the target is ready or timeout expired try: - signal.signal(signal.SIGINT, sig_int_handler) - ret = 0 - while not self.tg.is_target_ready_for_new_data(): - time.sleep(0.01) - now = datetime.datetime.now() - if (now - t0).total_seconds() > self._hc_timeout: - print('\n***DBG: FBK timeout') - self.lg.log_target_feedback_from( - '*** Timeout! The target does not seem to be ready.', - now, status_code=-1, source='Fuddly FmK' - ) - ret = -1 - self.tg.cleanup() - break + for tg in self.targets.values(): + while not tg.is_target_ready_for_new_data(): + time.sleep(0.005) + now = datetime.datetime.now() + if (now - t0).total_seconds() > self._hc_timeout_max: + self.lg.log_target_feedback_from( + source=FeedbackSource(self), + content='*** Timeout! The target {!s} does not seem to be ready.' + .format(self.available_targets_desc[tg]), + status_code=-1, + timestamp=now + ) + go_on = self._recover_target(tg) + ret = 0 if go_on else -1 + # tg.cleanup() + break except KeyboardInterrupt: self.lg.log_comment("*** Waiting for target to become ready has been cancelled by the user!\n") self.set_error("Waiting for target to become ready has been cancelled by the user!", code=Error.OperationCancelled) ret = -2 - self.tg.cleanup() + if tg: + tg.cleanup() except: self._handle_user_code_exception() ret = -3 - self.tg.cleanup() + if tg: + tg.cleanup() finally: signal.signal(signal.SIGINT, signal.SIG_IGN) @@ -2249,15 +2588,19 @@ def fmkdb_fetch_data(self, start_id=1, end_id=-1): data.set_data_model(dm) self.__register_in_data_bank(None, data) - @EnforceOrder(accepted_states=['S2']) + def _log_fmk_info(self, msg): + if self.lg: + self.lg.log_fmk_info(msg, do_record=False) + else: + print(colorize('*** [ {:s} ] ***'.format(msg), rgb=Color.FMKINFO)) + def enable_fmkdb(self): self.fmkDB.enable() - self.lg.log_fmk_info('Enable FmkDB', do_record=False) + self._log_fmk_info('Enable FmkDB') - @EnforceOrder(accepted_states=['S2']) def disable_fmkdb(self): self.fmkDB.disable() - self.lg.log_fmk_info('Disable FmkDB', do_record=False) + self._log_fmk_info('Disable FmkDB') @EnforceOrder(accepted_states=['S2']) def get_last_data(self): @@ -2460,7 +2803,16 @@ def show_operators(self): @EnforceOrder(accepted_states=['S2']) - def launch_operator(self, name, user_input=UserInputContainer(), use_existing_seed=True, verbose=False): + def get_operator(self, name): + operator = self.prj.get_operator(name) + if operator is None: + self.set_error('Invalid operator', code=Error.InvalidOp) + return None + else: + return operator + + @EnforceOrder(accepted_states=['S2']) + def launch_operator(self, name, user_input=None, use_existing_seed=True, verbose=False): operator = self.prj.get_operator(name) if operator is None: @@ -2470,7 +2822,7 @@ def launch_operator(self, name, user_input=UserInputContainer(), use_existing_se self.__reset_fmk_internals(reset_existing_seed=(not use_existing_seed)) try: - ok = operator._start(self._exportable_fmk_ops, self.dm, self.mon, self.tg, self.lg, user_input) + ok = operator._start(self._exportable_fmk_ops, self.dm, self.mon, self.targets, self.lg, user_input) except: self._handle_user_code_exception('Operator has crashed during its start() method') return False @@ -2489,7 +2841,7 @@ def launch_operator(self, name, user_input=UserInputContainer(), use_existing_se try: operation = operator.plan_next_operation(self._exportable_fmk_ops, self.dm, - self.mon, self.tg, self.lg, fmk_feedback) + self.mon, self.targets, self.lg, fmk_feedback) except: self._handle_user_code_exception('Operator has crashed during its plan_next_operation() method') return False @@ -2512,13 +2864,15 @@ def launch_operator(self, name, user_input=UserInputContainer(), use_existing_se instr_list = operation.get_instructions() for idx, instruction in enumerate(instr_list): - action_list, orig = instruction + action_list, orig, tg_ids = instruction if action_list is None: data = orig else: data = self.get_data(action_list, data_orig=orig, save_seed=use_existing_seed) + if data: + data.tg_ids = tg_ids data_list.append(data) @@ -2560,10 +2914,13 @@ def launch_operator(self, name, user_input=UserInputContainer(), use_existing_se if not data_list: continue - data_list = self._send_data(data_list, add_preamble=True) + data_list = self._send_data(data_list) if self._sending_error: self.lg.log_fmk_info("Operator will shutdown because of a sending error") break + elif self._stop_sending: + self.lg.log_fmk_info("Operator will shutdown because a DataProcess has yielded") + break elif data_list is None: self.lg.log_fmk_info("Operator will shutdown because there is no data to send") break @@ -2576,7 +2933,7 @@ def launch_operator(self, name, user_input=UserInputContainer(), use_existing_se multiple_data = len(data_list) > 1 try: - linst = operator.do_after_all(self._exportable_fmk_ops, self.dm, self.mon, self.tg, self.lg) + linst = operator.do_after_all(self._exportable_fmk_ops, self.dm, self.mon, self.targets, self.lg) except: self._handle_user_code_exception('Operator has crashed during its .do_after_all() method') return False @@ -2600,15 +2957,6 @@ def launch_operator(self, name, user_input=UserInputContainer(), use_existing_se elif ret == -3: self.lg.log_fmk_info("Operator will shutdown because of exception in user code") - ack_date = self.tg.get_last_target_ack_date() - self.lg.log_target_ack_date(ack_date) - - # Delay introduced after logging data - if not self.__delay_fuzzing(): - exit_operator = True - self.lg.log_fmk_info("Operator will shutdown because waiting has been cancelled by the user") - - # Target fbk is logged only at the end of a burst if self._burst_countdown == self._burst: cont1 = self.log_target_feedback() @@ -2629,9 +2977,8 @@ def launch_operator(self, name, user_input=UserInputContainer(), use_existing_se op_status = linst.get_operator_status() op_tstamp = linst.get_timestamp() if op_feedback or op_status: - self.lg.log_operator_feedback(op_feedback, op_tstamp, - op_name=operator.__class__.__name__, - status_code=op_status) + self.lg.log_operator_feedback(operator=operator, content=op_feedback, + status_code=op_status, timestamp=op_tstamp) comments = linst.get_comments() if comments: @@ -2640,12 +2987,20 @@ def launch_operator(self, name, user_input=UserInputContainer(), use_existing_se if op_status is not None and op_status < 0: exit_operator = True self.lg.log_fmk_info("Operator will shutdown because it returns a negative status") - self._recover_target() + for tg in self.targets.values(): + self._recover_target(tg) if self._burst_countdown == self._burst: - self.tg.cleanup() + for tg in self.targets.values(): + tg.cleanup() + + # Delay introduced after logging data + if not self.__delay_fuzzing(): + exit_operator = True + self.lg.log_fmk_info("Operator will shutdown because waiting has been cancelled by the user") + try: - operator.stop(self._exportable_fmk_ops, self.dm, self.mon, self.tg, self.lg) + operator.stop(self._exportable_fmk_ops, self.dm, self.mon, self.targets, self.lg) except: self._handle_user_code_exception('Operator has crashed during its stop() method') return False @@ -2657,11 +3012,11 @@ def launch_operator(self, name, user_input=UserInputContainer(), use_existing_se @EnforceOrder(accepted_states=['S2']) def get_data(self, action_list, data_orig=None, valid_gen=False, save_seed=False): ''' - @action_list shall have the following formats: - [(action_1, generic_UI_1, specific_UI_1), ..., - (action_n, generic_UI_n, specific_UI_n)] + @action_list shall have a format compatible with what follows: + [(action_1, UserInput_1), ..., + (action_n, UserInput_n)] - [action_1, (action_2, generic_UI_2, specific_UI_2), ... action_n] + [action_1, (action_2, UserInput_2), ... action_n] where action_N can be either: dmaker_type_N or (dmaker_type_N, dmaker_name_N) ''' @@ -2708,21 +3063,14 @@ def get_data(self, action_list, data_orig=None, valid_gen=False, save_seed=False unrecoverable_error = False activate_all = False - for full_action, idx in zip(action_list, range(len(action_list))): + for idx, full_action in enumerate(action_list): if isinstance(full_action, (tuple, list)): - if len(full_action) == 2: - action, gen_args = full_action - user_input = UserInputContainer(generic=gen_args, specific=None) - elif len(full_action) == 3: - action, gen_args, args = full_action - user_input = UserInputContainer(generic=gen_args, specific=args) - else: - print(full_action) - raise ValueError + assert len(full_action) == 2 + action, user_input = full_action else: action = full_action - user_input = UserInputContainer(generic=None, specific=None) + user_input = None if unrecoverable_error: break @@ -2888,7 +3236,7 @@ def get_data(self, action_list, data_orig=None, valid_gen=False, save_seed=False data = Data(dmaker_obj.produced_seed.get_content(do_copy=True)) else: data = dmaker_obj.generate_data(self.dm, self.mon, - self.tg) + self.targets) if save_seed and dmaker_obj.produced_seed is None: # Usefull to replay from the beginning a modelwalking sequence dmaker_obj.produced_seed = Data(data.get_content(do_copy=True)) @@ -2897,9 +3245,9 @@ def get_data(self, action_list, data_orig=None, valid_gen=False, save_seed=False if not self._is_data_valid(data): invalid_data = True else: - data = dmaker_obj.disrupt_data(self.dm, self.tg, data) + data = dmaker_obj.disrupt_data(self.dm, self.targets, data) elif isinstance(dmaker_obj, StatefulDisruptor): - # we only check validity in the case the stateful disruptor is + # we only check validity in the case the stateful disruptor # has not been seeded if dmaker_obj.is_attr_set(DataMakerAttr.NeedSeed) and not \ self._is_data_valid(data): @@ -2910,7 +3258,7 @@ def get_data(self, action_list, data_orig=None, valid_gen=False, save_seed=False data = ret dmaker_obj.set_attr(DataMakerAttr.NeedSeed) else: - data = dmaker_obj.disrupt_data(self.dm, self.tg, data) + data = dmaker_obj.disrupt_data(self.dm, self.targets, data) else: raise ValueError @@ -2958,6 +3306,8 @@ def _handle_disruptors_handover(dmlist): dmlist_mangled = dmlist[-2::-1] dmlist_mangled_size = len(dmlist_mangled) for dmobj, idx in zip(dmlist_mangled, range(dmlist_mangled_size)): + # if save_seed and isinstance(dmobj, Generator): + # dmobj.produced_seed = None if dmobj.is_attr_set(DataMakerAttr.Controller): dmobj.set_attr(DataMakerAttr.Active) if dmobj.is_attr_set(DataMakerAttr.HandOver): @@ -3011,6 +3361,11 @@ def _handle_disruptors_handover(dmlist): @EnforceOrder(accepted_states=['S1','S2']) def cleanup_all_dmakers(self, reset_existing_seed=True): + return self._cleanup_all_dmakers(reset_existing_seed=reset_existing_seed) + + def _cleanup_all_dmakers(self, reset_existing_seed=True): + if not self.__initialized_dmakers: + return for dmaker_obj in self.__initialized_dmakers: if self.__initialized_dmakers[dmaker_obj][0]: @@ -3100,7 +3455,7 @@ def show_probes(self): self.lg.print_console('') for p in probes: try: - status = self.mon.get_probe_status(p).get_status() + status = self.mon.get_probe_status(p).value except: status = None msg = "name: %s (status: %s, delay: %f) --> " % \ @@ -3251,12 +3606,8 @@ def _make_str(k, v): msg = '\n' + colorize(obj.__doc__, rgb=Color.INFO_ALT_HLIGHT) else: msg = '' - if obj._gen_args_desc: - msg += "\n generic args: " - for k, v in obj._gen_args_desc.items(): - msg += _make_str(k, v) if obj._args_desc: - msg += "\n specific args: " + msg += "\n parameters: " for k, v in obj._args_desc.items(): msg += _make_str(k, v) @@ -3370,19 +3721,30 @@ class FmkShell(cmd.Cmd): def __init__(self, title, fmk_plumbing, completekey='tab', stdin=None, stdout=None): cmd.Cmd.__init__(self, completekey, stdin, stdout) self.fz = fmk_plumbing - self.prompt = '>> ' self.intro = colorize(FontStyle.BOLD + "\n-=[ %s ]=- (with Fuddly FmK %s)\n" % (title, fuddly_version), rgb=Color.TITLE) self.__allowed_cmd = re.compile( - '^quit$|^show_projects$|^show_data_models$|^load_project|^load_data_model|^set_target|^show_targets$|^launch$' \ - '|^run_project|^display_color_theme$|^help' + '^quit$|^show_projects$|^show_data_models$|^load_project|^load_data_model|^load_targets|^show_targets$|^launch$' \ + '|^run_project|^config|^display_color_theme$|^fmkdb_disable$|^fmkdb_enable$|^help' ) - self.dmaker_name_re = re.compile('([#\-\w]+)(.*)', re.S) - # the symbol '<' shall not be used within group(3) - self.input_gen_arg_re = re.compile('<(.*)>(.*)', re.S) - self.input_spe_arg_re = re.compile('\((.*)\)', re.S) - self.input_arg_re = re.compile('(.*)=(.*)', re.S) + self.dmaker_name_re = re.compile('^([#\-\w]+)(\(?[^\(\)]*\)?)$', re.S) + self.input_params_re = re.compile('\((.*)\)', re.S) + self.input_param_values_re = re.compile('(.*)=(.*)', re.S) + + self.config = config(self, path=[config_folder]) + def save_config(): + filename=os.path.join( + config_folder, + self.config.config_name + '.ini') + with open(filename, 'w') as cfile: + self.config.write(cfile) + atexit.register(save_config) + + self.prompt = self.config.prompt + ' ' + self.available_configs = { + "framework": self.fz.config, + "shell": self.config} self.__error = False self.__error_msg = '' @@ -3403,6 +3765,8 @@ def save_history(history_path=history_path): def postcmd(self, stop, line): + self.prompt = self.config.prompt + ' ' + if self._quit_shell: self._quit_shell = False msg = colorize(FontStyle.BOLD + "\nReally Quit? [Y/n]", rgb=Color.WARNING) @@ -3494,6 +3858,178 @@ def do_display_color_theme(self, line): return False + def do_logger_switch_format(self, line): + ''' + Change the way the logger display the data which are sent to the targets and retrieved from them. + (From raw format to interpreted format and reversely.) + This command modify the current Project's Logger. + ''' + self.fz.lg.export_raw_data = not self.fz.lg.export_raw_data + + return False + + def complete_config(self, text, line, bgidx, endix, target=None): + init = False + if target is None: + init = True + + args = line.split() + if args[-1] == text: + args.pop() + if init: + if len(args) == 1: + comp = [k for k in self.available_configs.keys()] + if text != '': + comp = [i for i in comp if i.startswith(text)] + return comp + + try: + if text != '': + return self.complete_config( + text, + ' '.join(['config'] + args[2:] + [text]), + 0, + 0, + self.available_configs[args[1]]) + else: + return self.complete_config( + '', + ' '.join(['config'] + args[2:]), + 0, + 0, + self.available_configs[args[1]]) + except KeyError: + pass + + return [] + + if len(args) == 1 and isinstance(target, config): + comp = (target.parser.options('global') + + target.parser.sections()) + if text != '': + comp = [i for i in comp if i.startswith(text)] + comp = [i.replace('.', ' ') for i in comp if ( + i[-4:] != '.doc' and i != 'config_name' and i != 'global')] + return comp + if len(args) > 1 and args[1] == 'shell': + return self.complete_config( + text, + ' '.join(args[1:] + [text]), + 0, + 0, + self.config) + if len(args) > 1 and target.parser.has_section(args[1]): + return self.complete_config( + text, + ' '.join(args[1:] + [text]), + 0, + 0, + getattr(target, args[1])) + comp = target.parser.options('global') + comp = [i for i in comp if i.startswith(args[-1] + '.')] + comp = [i[len(args[-1]) + 1:] for i in comp if ( + i[-4:] != '.doc' and i != 'config_name' and i != 'global')] + if text != '': + comp = [i for i in comp if i.startswith(text)] + return comp + + def do_config(self, line, target=None): + '''Get and set miscellaneous options + + Usage: + - config + List all configuration options available. + - config [name [subname...]] + Get value associated with . + - config [name [subname...]] value + Set value associated with . + ''' + self.__error = True + + level = self.config.config.indent.level + indent = self.config.config.indent.width + middle = self.config.config.middle + + args = line.split() + if target is None: + if len(args) == 0: + print('Available configurations:') + for target in self.available_configs: + print(' - {}'.format(target)) + print('\n\t > Type "config " to display documentation.') + self.__error = False + return False + else: + try: + target = self.available_configs[args[0]] + self.__error = False + return self.do_config(' '.join(args[1:]), target) + except KeyError as e: + print('Unknown config "{}": '.format(args[0]) + str(e)) + return True + + if len(args) == 0: + print(target.help(None, level, indent, middle)) + self.__error = False + return False + elif len(args) == 1: + print(target.help(args[0], level, indent, middle)) + self.__error = False + return False + + section = args[0] + try: + attr = getattr(target, section) + except: + self.__error_msg = ( + "'{}' is not a valid config key".format(section)) + return False + + if isinstance(attr, config): + self.__error = False + return self.do_config(' '.join(args[1:]), attr) + + if len(args) == 2: + if isinstance(attr, config_dot_proxy): + self.__error = False + key = '.'.join(args) + print(target.help(key, level, indent, middle)) + self.__error = False + return False + + try: + setattr(target, args[0], args[1]) + except AttributeError as e: + self.__error_msg = 'config: ' + str(e) + return False + + print(target.help(args[0], level, indent, middle)) + self.__error = False + return False + + if isinstance(attr, config_dot_proxy): + key = '.'.join(args[:-1]) + try: + attr = getattr(target, key) + except: + self.__error_msg = ( + "'{}' is not a valid config key".format(key)) + return False + + try: + setattr(target, key, args[-1]) + except AttributeError as e: + self.__error_msg = 'config: ' + str(e) + return False + + print(target.help(key, level, indent, middle)) + self.__error = False + return False + + self.__error_msg = ( + "'{}' do not have subkeys".format(args[0])) + return False + def do_load_data_model(self, line): '''Load a Data Model by name''' self.__error = True @@ -3577,7 +4113,7 @@ def do_run_project(self, line): 2. Load the default data model of the project file 3. Launch the project by starting fuddly subsystems - |_ syntax: run_project [target_number] + |_ syntax: run_project [target_id1 ... target_idN] ''' self.__error = True @@ -3585,7 +4121,7 @@ def do_run_project(self, line): args = line.split() args_len = len(args) - if args_len < 1 or args_len > 2: + if args_len < 1: self.__error_msg = "Syntax Error!" return False @@ -3596,11 +4132,15 @@ def do_run_project(self, line): tg_id = None if tg_id: + tg_ids = [] try: - tg_id = int(tg_id) + for tg_id in args[1:]: + tg_ids.append(int(tg_id)) except ValueError: - self.__error_msg = "Parameter 2 shall be an integer!" + self.__error_msg = "Parameter N (N>=2) shall be an integer!" return False + else: + tg_ids = None ok = False for prj in self.fz.projects(): @@ -3613,7 +4153,7 @@ def do_run_project(self, line): return False self.__error_msg = "Unable to launch the project '%s'" % prj_name - if not self.fz.run_project(prj=prj, tg=tg_id): + if not self.fz.run_project(prj=prj, tg_ids=tg_ids): return False self.__error = False @@ -3621,24 +4161,27 @@ def do_run_project(self, line): - def do_set_target(self, line): + def do_load_targets(self, line): ''' Set the target number to use - |_ syntax: set_target + |_ syntax: load_targets [target_id2 ... target_idN] ''' self.__error = True args = line.split() args_len = len(args) - if args_len != 1: + if args_len < 1: return False + + tg_ids = [] try: - num = int(args[0]) + for tg_id in args: + tg_ids.append(int(tg_id)) except ValueError: return False - self.fz.set_target(num) + self.fz.load_targets(tg_ids) self.__error = False return False @@ -3657,6 +4200,11 @@ def do_show_fmk_internals(self, line): return False + def do_show_knowledge(self, line): + '''Show the current status of knowledge''' + self.fz.show_knowledge() + + return False def do_launch(self, line): '''Launch the loaded project by starting every needed components''' @@ -3810,7 +4358,7 @@ def do_disable_wkspace(self, line): def do_send_valid(self, line): ''' Build a data in multiple step from a valid source - |_ syntax: send_fullvalid [disruptor_type_1 ... disruptor_type_n] + |_ syntax: send_valid [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] |_ Note: generator_type shall have at least one valid generator ''' ret = self.do_send(line, valid_gen=True) @@ -3818,20 +4366,26 @@ def do_send_valid(self, line): def do_send_loop_valid(self, line): ''' - Loop ( Build a data in multiple step from a valid source ) - |_ syntax: send_loop_valid <#loop> [disruptor_type_1 ... disruptor_type_n] + Execute the 'send_valid' command in a loop + |_ syntax: send_loop_valid <#loop> [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] |_ Note: generator_type shall have at least one valid generator ''' ret = self.do_send_loop(line, valid_gen=True) return ret - def do_send_loop_noseed(self, line): + def do_send_loop_keepseed(self, line): ''' - Loop ( Build a data in multiple step from a valid source ) - |_ syntax: send_loop_noseed <#loop> [disruptor_type_1 ... disruptor_type_n] - |_ Note: generator_type shall have at least one valid generator + Execute the 'send' command in a loop and save the seed + |_ syntax: send_loop_keepseed <#loop> [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] + + Notes: + - To loop indefinitely use -1 for #loop. To stop the loop use Ctrl+C + - send_loop_keepseed keep the generator output until a reset is performed on it. + Thus, in the context of a disruptor chain, if the generator is non-deterministic, + and even if you clean up the generator, you could still reproduce the exact sequence + of data production from the beginning ''' - ret = self.do_send_loop(line, use_existing_seed=False) + ret = self.do_send_loop(line, use_existing_seed=True) return ret @@ -3879,7 +4433,7 @@ def do_set_disruptor_weight(self, line): return False - def do_launch_operator(self, line, use_existing_seed=True, verbose=False): + def do_launch_operator(self, line, use_existing_seed=False, verbose=False): ''' Launch the specified operator and use any existing seed |_ syntax: launch_operator @@ -3898,7 +4452,7 @@ def do_launch_operator(self, line, use_existing_seed=True, verbose=False): return False operator = t[0][0] - user_input = UserInputContainer(generic=t[0][1], specific=t[0][2]) + user_input = t[0][1] self.fz.launch_operator(operator, user_input, use_existing_seed=use_existing_seed, verbose=verbose) @@ -3907,12 +4461,12 @@ def do_launch_operator(self, line, use_existing_seed=True, verbose=False): return False - def do_launch_operator_noseed(self, line): + def do_launch_operator_keepseed(self, line): ''' Launch the specified operator without using any current seed - |_ syntax: launch_operator_noseed + |_ syntax: launch_operator_keepseed ''' - ret = self.do_launch_operator(line, use_existing_seed=False) + ret = self.do_launch_operator(line, use_existing_seed=True) return ret @@ -3933,12 +4487,12 @@ def do_show_operators(self, line): def __parse_instructions(self, cmdline): ''' return a list of the following format: - [(action_1, [gen_arg_11, ..., gen_arg_1n], [arg_11, ..., arg_1n]), ..., - (action_n, [gen_arg_n1, ..., gen_arg_nn], [arg_n1, ..., arg_nn])] + [(action_1, [arg_11, ..., arg_1n]), ..., + (action_n, [arg_n1, ..., arg_nn])] ''' def __extract_arg(exp, dico): - re_obj = self.input_arg_re.match(exp) + re_obj = self.input_param_values_re.match(exp) if re_obj is None: return False key, val = re_obj.group(1), re_obj.group(2) @@ -3957,30 +4511,11 @@ def __extract_arg(exp, dico): name = parsed.group(1) allargs_str = parsed.group(2) else: - name = 'INCORRECT_NAME' - allargs_str = None + return None if allargs_str is not None: - parsed = self.input_gen_arg_re.match(allargs_str) - # Parse generic arguments - if parsed: - arg_str = parsed.group(1) - specific_args_str = parsed.group(2) - gen_args = {} - l = arg_str.split(':') - for a in l: - ok = __extract_arg(a, gen_args) - if not ok: - return None - else: - gen_args = None - specific_args_str = allargs_str - - # Parse specific arguments - if specific_args_str is not None: - parsed = self.input_spe_arg_re.match(specific_args_str) - else: - parsed = None + # Parse arguments + parsed = self.input_params_re.match(allargs_str) if parsed: arg_str = parsed.group(1) args = {} @@ -3993,19 +4528,29 @@ def __extract_arg(exp, dico): args = None else: - gen_args = None args = None - gen_ui = UI() - spe_ui = UI() - if gen_args is not None and len(gen_args) > 0: - gen_ui.set_user_inputs(gen_args) + user_input = UI() if args is not None and len(args) > 0: - spe_ui.set_user_inputs(args) + user_input.set_user_inputs(args) - d.append((name, gen_ui, spe_ui)) + d.append((name, user_input)) - return d + return d if bool(d) else None + + def _retrieve_tg_ids(self, args): + tg_ids = [] + try: + for arg in args[::-1]: + tg_id = int(arg) + tg_ids.append(tg_id) + args = [] + except ValueError: + if tg_ids: + tg_ids = tg_ids[::-1] + args = args[:-len(tg_ids)] + + return args, tg_ids def do_reload_data_model(self, line): ''' @@ -4019,21 +4564,24 @@ def do_reload_data_model(self, line): def do_reload_all(self, line): ''' Reload the current data model and all its associated components (target, monitor, logger) - |_ syntax: reload_all [target_number] + |_ syntax: reload_all [target_id1 ... target_idN] ''' self.__error = True args = line.split() args_len = len(args) - num = None if args_len > 0: + tg_ids = [] try: - num = int(args[0]) + for tg_id in args: + tg_ids.append(int(tg_id)) except ValueError: return False + else: + tg_ids = None - self.fz.reload_all(tg_num=num) + self.fz.reload_all(tg_ids=tg_ids) self.__error = False return False @@ -4094,28 +4642,40 @@ def do_reset_dmaker(self, line): return ret + def do_flush_feedback(self, line): + ''' + Collect the residual feedback (received by the target and the probes) + ''' + self.fz.collect_residual_feedback() + return False + + def do_send(self, line, valid_gen=False, verbose=False): ''' Carry out multiple fuzzing steps in sequence - |_ syntax: send [disruptor_type_1 ... disruptor_type_n] + |_ syntax: send [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] ''' self.__error = True args = line.split() + args_len = len(args) - if len(args) < 1: + if args_len < 1: return False + args, tg_ids = self._retrieve_tg_ids(args) + t = self.__parse_instructions(args) if t is None: self.__error_msg = "Syntax Error!" return False data = self.fz.get_data(t, valid_gen=valid_gen) - if data is None: return False + if tg_ids: + data.tg_ids = tg_ids self.fz.send_data_and_log(data, verbose=verbose) self.__error = False @@ -4125,25 +4685,30 @@ def do_send(self, line, valid_gen=False, verbose=False): def do_send_verbose(self, line): ''' Carry out multiple fuzzing steps in sequence (pretty print enabled) - |_ syntax: send_verbose [disruptor_type_1 ... disruptor_type_n] + |_ syntax: send_verbose [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] ''' ret = self.do_send(line, verbose=True) return ret - def do_send_loop(self, line, valid_gen=False, use_existing_seed=True): + def do_send_loop(self, line, valid_gen=False, use_existing_seed=False): ''' - Loop ( Carry out multiple fuzzing steps in sequence ) - |_ syntax: send_loop <#loop> [disruptor_type_1 ... disruptor_type_n] + Execute the 'send' command in a loop + |_ syntax: send_loop <#loop> [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] - Note: To loop indefinitely use -1 for #loop. To stop the loop use Ctrl+C + Notes: + - To loop indefinitely use -1 for #loop. To stop the loop use Ctrl+C ''' args = line.split() + args_len = len(args) self.__error = True - if len(args) < 2: + if args_len < 2: return False + + args, tg_ids = self._retrieve_tg_ids(args) + try: max_loop = int(args.pop(0)) if max_loop < 2 and max_loop != -1: @@ -4156,17 +4721,29 @@ def do_send_loop(self, line, valid_gen=False, use_existing_seed=True): self.__error_msg = "Syntax Error!" return False - # for i in range(nb): - cpt = 0 - while cpt < max_loop or max_loop == -1: - cpt += 1 - data = self.fz.get_data(t, valid_gen=valid_gen, save_seed=use_existing_seed) - if data is None: - return False - cont = self.fz.send_data_and_log(data) - if not cont: - break - + conf = self.config.send_loop.aligned_options + kwargs = { + 'enabled': self.config.send_loop.aligned, + 'page_head': r'^[^=]+====. [^ ]+ .==. [^=]+={9,}.{4}$', + 'batch_mode': (max_loop == -1) and conf.batch_mode, + 'hide_cursor': conf.hide_cursor, + 'prompt_height': conf.prompt_height + } + + with aligned_stdout(**kwargs): + # for i in range(nb): + cpt = 0 + while cpt < max_loop or max_loop == -1: + cpt += 1 + data = self.fz.get_data(t, valid_gen=valid_gen, save_seed=use_existing_seed) + if data is None: + return False + if tg_ids: + data.tg_ids = tg_ids + cont = self.fz.send_data_and_log(data) + if not cont: + break + self.__error = False return False @@ -4174,15 +4751,17 @@ def do_send_loop(self, line, valid_gen=False, use_existing_seed=True): def do_send_with(self, line): ''' Generate data from specific generator - |_ syntax: send_with + |_ syntax: send_with [targetID1 ... targetIDN] ''' self.__error = True args = line.split() - if len(args) != 2: + if len(args) < 2: return False + args, tg_ids = self._retrieve_tg_ids(args) + t = self.__parse_instructions([args[0]])[0] if t is None: self.__error_msg = "Syntax Error!" @@ -4193,6 +4772,8 @@ def do_send_with(self, line): if data is None: return False + if tg_ids: + data.tg_ids = tg_ids self.fz.send_data_and_log(data) self.__error = False @@ -4202,14 +4783,17 @@ def do_send_with(self, line): def do_send_loop_with(self, line): ''' Loop ( Generate data from specific generator ) - |_ syntax: send_loop_with <#loop> + |_ syntax: send_loop_with <#loop> [targetID1 ... targetIDN] ''' self.__error = True args = line.split() - if len(args) != 3: + if len(args) < 3: return False + + args, tg_ids = self._retrieve_tg_ids(args) + try: nb = int(args[0]) except ValueError: @@ -4222,12 +4806,24 @@ def do_send_loop_with(self, line): action = [((t[0], args[2]), t[1])] - for i in range(nb): - data = self.fz.get_data(action) - if data is None: - return False + conf = self.config.send_loop.aligned_options + kwargs = { + 'enabled': self.config.send_loop.aligned, + 'page_head': r'^[^=]+====. [^ ]+ .==. [^=]+={9,}.{4}$', + 'batch_mode': False, + 'hide_cursor': conf.hide_cursor, + 'prompt_height': conf.prompt_height + } - self.fz.send_data_and_log(data) + with aligned_stdout(**kwargs): + for i in range(nb): + data = self.fz.get_data(action) + if data is None: + return False + + if tg_ids: + data.tg_ids = tg_ids + self.fz.send_data_and_log(data) self.__error = False return False @@ -4236,7 +4832,7 @@ def do_send_loop_with(self, line): def do_multi_send(self, line): ''' - Send multi-data to a target. Generation instructions must be provided when + Send several data to one or more targets. Generation instructions must be provided when requested (same format as the command 'send'). |_ syntax: multi_send [#loop] ''' @@ -4258,7 +4854,7 @@ def do_multi_send(self, line): while True: idx += 1 - msg = "*** Data generation instructions [#%d] (type '!' when all instructions are provided):\n" % idx + msg = "*** Data generation instructions [#{:d}] (type '!' when all instructions are provided):\n".format(idx) if sys.version_info[0] == 2: actions_str = raw_input(msg) else: @@ -4272,12 +4868,13 @@ def do_multi_send(self, line): if len(l) < 1: return False + l, tg_ids = self._retrieve_tg_ids(l) actions = self.__parse_instructions(l) if actions is None: self.__error_msg = "Syntax Error!" return False - actions_list.append(actions) + actions_list.append((actions, tg_ids)) prev_data_list = None exhausted_data_cpt = 0 @@ -4292,7 +4889,10 @@ def do_multi_send(self, line): exhausted_data[j] = False if not exhausted_data[j]: - data = self.fz.get_data(actions_list[j]) + action_seq, tg_ids = actions_list[j] + data = self.fz.get_data(action_seq) + if tg_ids and data is not None: + data.tg_ids = tg_ids else: if prev_data_list is not None: data = prev_data_list[j] @@ -4330,24 +4930,34 @@ def do_multi_send(self, line): def do_set_feedback_timeout(self, line): ''' Set the time duration for feedback gathering (if supported by the target) - | syntax: set_feedback_timeout + | syntax: set_feedback_timeout [targetID] | |_ possible values for : | 0 : no timeout | x>0 : timeout expressed in seconds (fraction is possible) + | |_ if targetID is not provided, the value applies to all enabled targets ''' self.__error = True args = line.split() args_len = len(args) - if args_len != 1: + if 3 > args_len < 1: return False try: timeout = float(args[0]) - self.fz.set_feedback_timeout(timeout) except: return False + tg_id = None + if args_len > 1: + try: + tg_id = int(args[1]) + except ValueError: + self.__error_msg = "Parameter 2 shall be an integer!" + return False + + self.fz.set_feedback_timeout(timeout, tg_id=tg_id) + self.__error = False return False @@ -4356,11 +4966,28 @@ def do_switch_feedback_mode(self, line): Switch target feedback mode between: - wait for the full time slot allocated for feedback retrieval - wait until the target has send something back to us + + Syntax: switch_feedback_mode ''' - self.fz.switch_feedback_mode(do_record=True, do_show=True) + self.__error = True + + args = line.split() + args_len = len(args) + + if args_len != 1: + return False + + try: + tg_id = int(args[0]) + except ValueError: + return False + + self.fz.switch_feedback_mode(tg_id, do_record=True, do_show=True) + + self.__error = False return False - def do_set_health_timeout(self, line): + def do_set_health_check_timeout(self, line): ''' Set the timeout when the FMK checks the target readiness (Default = 10). | syntax: set_health_timeout @@ -4464,18 +5091,20 @@ def do_empty_wkspace(self, line): def do_replay_db(self, line): ''' Replay data from the Data Bank and optionnaly apply new disruptors on it - |_ syntax: replay_db [disruptor_type_1 ... disruptor_type_n] + |_ syntax: replay_db i [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] ''' self.__error = True args = line.split() + args, tg_ids = self._retrieve_tg_ids(args) args_len = len(args) if args_len < 1: return False + try: - idx = int(args.pop(0)) + idx = int(args.pop(0)[1:]) except ValueError: return False @@ -4497,6 +5126,8 @@ def do_replay_db(self, line): self.__error = False + if tg_ids: + data.tg_ids = tg_ids self.fz.send_data_and_log(data, original_data=data_orig) return False @@ -4505,19 +5136,22 @@ def do_replay_db(self, line): def do_replay_db_loop(self, line): ''' Loop ( Replay data from the Data Bank and optionnaly apply new disruptors on it ) - |_ syntax: replay_db_loop <#loop> [disruptor_type_1 ... disruptor_type_n] + |_ syntax: replay_db_loop <#loop> i [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] ''' self.__error = True args = line.split() + args, tg_ids = self._retrieve_tg_ids(args) + args_len = len(args) if args_len < 2: return False + try: nb = int(args.pop(0)) - idx = int(args.pop(0)) + idx = int(args.pop(0)[1:]) except ValueError: return False @@ -4538,10 +5172,14 @@ def do_replay_db_loop(self, line): if new_data is None: return False + if tg_ids: + new_data.tg_ids = tg_ids self.fz.send_data_and_log(new_data, original_data=data_orig) else: for i in range(nb): + if tg_ids: + data.tg_ids = tg_ids self.fz.send_data_and_log(data, original_data=data_orig) self.__error = False @@ -4550,7 +5188,13 @@ def do_replay_db_loop(self, line): def do_replay_db_all(self, line): - '''Replay all data from the Data Bank''' + ''' + Replay all data from the Data Bank + |_ syntax: replay_db_all [targetID1 ... targetIDN] + ''' + + args = line.split() + args, tg_ids = self._retrieve_tg_ids(args) try: next(self.fz.iter_data_bank()) @@ -4560,6 +5204,8 @@ def do_replay_db_all(self, line): return False for data_orig, data in self.fz.iter_data_bank(): + if tg_ids: + data.tg_ids = tg_ids self.fz.send_data_and_log(data, original_data=data_orig) return False @@ -4627,7 +5273,7 @@ def do_show_scenario(self, line): def do_replay_last(self, line): ''' Replay last data and optionnaly apply new disruptors on it - |_ syntax: replay_last [disruptor_type_1 ... disruptor_type_n] + |_ syntax: replay_last [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] ''' self.__error = True @@ -4636,9 +5282,12 @@ def do_replay_last(self, line): if data is None: return False + tg_ids = None + if line: args = line.split() data_orig = data + args, tg_ids = self._retrieve_tg_ids(args) t = self.__parse_instructions(args) if t is None: @@ -4651,6 +5300,8 @@ def do_replay_last(self, line): self.__error = False + if tg_ids: + data.tg_ids = tg_ids self.fz.send_data_and_log(data, original_data=data_orig) return False @@ -4662,9 +5313,22 @@ def do_send_raw(self, line): |_ syntax: send_raw ''' + self.__error_msg = "Syntax Error!" + args = line.split() + args_len = len(args) + + if args_len < 1: + self.__error = True + return False + + args, tg_ids = self._retrieve_tg_ids(args) + line = ''.join(args) + if line: data = Data(line) - + + if tg_ids: + data.tg_ids = tg_ids self.fz.send_data_and_log(data, None) else: self.__error = True @@ -4676,6 +5340,16 @@ def do_send_eval(self, line): Send python-evaluation of the parameter |_ syntax: send_eval ''' + self.__error_msg = "Syntax Error!" + args = line.split() + args_len = len(args) + + if args_len < 1: + self.__error = True + return False + + args, tg_ids = self._retrieve_tg_ids(args) + line = ''.join(args) if line: try: @@ -4684,6 +5358,8 @@ def do_send_eval(self, line): self.__error = True return False + if tg_ids: + data.tg_ids = tg_ids self.fz.send_data_and_log(data, None) else: self.__error = True diff --git a/framework/project.py b/framework/project.py index a0a7574..9dfd83d 100644 --- a/framework/project.py +++ b/framework/project.py @@ -22,19 +22,92 @@ ################################################################################ from __future__ import print_function + +try: + import queue as queue +except: + import Queue as queue + from framework.monitor import * +from framework.knowledge.feedback_handler import * +from framework.knowledge.information import InformationCollector +from framework.value_types import VT +from framework.node import Env +from framework.data_model import DataModel +from framework.tactics_helpers import DataMaker +from framework.scenario import ScenarioEnv class Project(object): name = None default_dm = None - def __init__(self): + feedback_gate = None + + def __init__(self, enable_fbk_processing=True): self.monitor = Monitor() - self.target = None + self._knowledge_source = InformationCollector() + self._fbk_processing_enabled = enable_fbk_processing + self._feedback_processing_thread = None + self._fbk_handlers = [] + + self.scenario_target_mapping = None + self.reset_target_mappings() + + self.project_scenarios = None + self.targets = None self.dm = None self.operators = {} + ################################ + ### Knowledge Infrastructure ### + ################################ + + @property + def knowledge_source(self): + return self._knowledge_source + + def add_knowledge(self, *info): + self.knowledge_source.add_information(info, initial_trust_value=50) + + def reset_knowledge(self): + self.knowledge_source.reset_information() + + def register_feedback_handler(self, fbk_handler): + self._fbk_handlers.append(fbk_handler) + + def notify_data_sending(self, data_list, timestamp, target): + for fh in self._fbk_handlers: + fh.notify_data_sending(data_list, timestamp, target) + + def trigger_feedback_handlers(self, source, timestamp, content, status): + if not self._fbk_processing_enabled: + return + self._feedback_fifo.put((source, timestamp, content, status)) + + def _feedback_processing(self): + ''' + core function of the feedback processing thread + ''' + while self._run_fbk_handling_thread: + try: + fbk_tuple = self._feedback_fifo.get(timeout=0.5) + except queue.Empty: + continue + + for fh in self._fbk_handlers: + info = fh.process_feedback(*fbk_tuple) + if info: + self.knowledge_source.add_information(info) + + def estimate_last_data_impact_uniqueness(self): + similarity = UNIQUE + if self._fbk_processing_enabled: + for fh in self._fbk_handlers: + similarity += fh.estimate_last_data_impact_uniqueness() + + return similarity + ##################### ### Configuration ### ##################### @@ -42,8 +115,8 @@ def __init__(self): def set_logger(self, logger): self.logger = logger - def set_target(self, target): - self.target = target + def set_targets(self, targets): + self.targets = targets def set_monitor(self, monitor): self.monitor = monitor @@ -51,7 +124,16 @@ def set_monitor(self, monitor): def set_data_model(self, dm): self.dm = dm - def register_new_operator(self, name, obj): + def map_targets_to_scenario(self, scenario_name, target_mapping): + self.scenario_target_mapping[scenario_name] = target_mapping + + def reset_target_mappings(self): + self.scenario_target_mapping = {} + + def register_scenarios(self, *scenarios): + self.project_scenarios = scenarios + + def register_operator(self, name, obj): if name in self.operators: print("\n*** /!\\ ERROR: The operator name '{:s}' is already used\n".format(name)) @@ -59,23 +141,44 @@ def register_new_operator(self, name, obj): self.operators[name] = obj - def register_new_probe(self, probe, blocking=False): + def register_probe(self, probe, blocking=False): try: self.monitor.add_probe(probe, blocking) except AddExistingProbeToMonitorError as e: print("\n*** /!\\ ERROR: The probe name '{:s}' is already used\n".format(e.probe_name)) raise ValueError - ########################## ### Runtime Operations ### ########################## def start(self): - pass + VT.knowledge_source = self.knowledge_source + Env.knowledge_source = self.knowledge_source + DataModel.knowledge_source = self.knowledge_source + DataMaker.knowledge_source = self.knowledge_source + ScenarioEnv.knowledge_source = self.knowledge_source + + if self._fbk_processing_enabled: + self._run_fbk_handling_thread = True + self._feedback_fifo = queue.Queue() + self._feedback_processing_thread = threading.Thread(target=self._feedback_processing, + name='fuddly feedback processing') + self._feedback_processing_thread.start() + def stop(self): - pass + VT.knowledge_source = None + Env.knowledge_source = None + DataModel.knowledge_source = None + DataMaker.knowledge_source = None + ScenarioEnv.knowledge_source = None + + if self._fbk_processing_enabled: + self._run_fbk_handling_thread = False + if self._feedback_processing_thread: + self._feedback_processing_thread.join() + self._feedback_fifo = None def get_operator(self, name): try: diff --git a/framework/scenario.py b/framework/scenario.py index 846f474..1b9ed55 100644 --- a/framework/scenario.py +++ b/framework/scenario.py @@ -32,7 +32,7 @@ from libs.utils import find_file, retrieve_app_handler class DataProcess(object): - def __init__(self, process, seed=None, auto_regen=False): + def __init__(self, process, seed=None, auto_regen=False, vtg_ids=None): """ Describe a process to generate a data. @@ -54,6 +54,8 @@ def __init__(self, process, seed=None, auto_regen=False): all disruptors are exhausted. If ``False``, the data process won't notify the framework to rerun the data maker chain, thus triggering the end of the scenario that embeds this data process. + vtg_ids (list): Virtual ID list of the targets to which the outcomes of this data process will be sent. + If ``None``, the outcomes will be sent to the first target that has been enabled. """ self.seed = seed self.auto_regen = auto_regen @@ -61,6 +63,7 @@ def __init__(self, process, seed=None, auto_regen=False): self.outcomes = None self.feedback_timeout = None self.feedback_mode = None + self.vtg_ids = vtg_ids self._process = [process] self._process_idx = 0 self._blocked = False @@ -152,15 +155,46 @@ class Step(object): def __init__(self, data_desc=None, final=False, fbk_timeout=None, fbk_mode=None, set_periodic=None, clear_periodic=None, step_desc=None, - do_before_data_processing=None, do_before_sending=None): + do_before_data_processing=None, do_before_sending=None, + valid=True, vtg_ids=None): + """ + Step objects are the building blocks of Scenarios. + + Args: + data_desc: + final: + fbk_timeout: + fbk_mode: + set_periodic: + clear_periodic: + step_desc: + do_before_data_processing: + do_before_sending: + valid: + vtg_ids (list, int): Virtual ID list of the targets to which the outcomes of this data process will be sent. + If ``None``, the outcomes will be sent to the first target that has been enabled. + If ``data_desc`` is a list, this parameter should be a list where each item is the ``vtg_ids`` + of the corresponding item in the ``data_desc`` list. + """ self.final = final + self.valid = valid self._step_desc = step_desc self._transitions = [] self._do_before_data_processing = do_before_data_processing self._do_before_sending = do_before_sending self._handle_data_desc(data_desc) + if vtg_ids is not None: + if isinstance(data_desc, list): + assert isinstance(vtg_ids, list) + assert len(vtg_ids) == len(data_desc) + self.vtg_ids_list = vtg_ids + else: + self.vtg_ids_list = [vtg_ids] + else: + self.vtg_ids_list = None + self.make_free() # need to be set after self._data_desc @@ -329,6 +363,8 @@ def content(self): self._atom = {} if update_node: self._atom[idx] = self._scenario_env.dm.get_atom(self._node_name[idx]) + self._data_desc[idx] = Data(self._atom[idx]) + update_node = False atom_list.append(self._atom[idx]) return atom_list[0] if len(atom_list) == 1 else atom_list @@ -359,7 +395,7 @@ def get_data(self): # Practically it means that the creation of these data need to be performed # by data framework callback (CallBackOps.Replace_Data) because # a generator (by which a scenario will be executed) can only provide one data. - d = Data('STEP:POISON_1') + d = Data('STEP:POISON_2') if not d.has_info(): if self._step_desc is not None: @@ -393,6 +429,11 @@ def get_data(self): if self._feedback_mode is not None: d.feedback_mode = self._feedback_mode + if self.vtg_ids_list: + # Note in the case of self._data_desc contains multiple data, related + # vtg_ids are retrieved directly from the Step in the Replace_Data callback. + d.tg_ids = self.vtg_ids_list[0] + return d @property @@ -435,7 +476,7 @@ def __str__(self): if self.__class__.__name__ != 'Step': step_desc += '[' + self.__class__.__name__ + ']' else: - step_desc += d.content.name if isinstance(d.content, Node) else 'Data(...)' + step_desc += d.content.name.upper() if isinstance(d.content, Node) else 'Data(...)' elif isinstance(d, str): step_desc += "{:s}".format(self._node_name[idx].upper()) else: @@ -490,17 +531,19 @@ def __copy__(self): class FinalStep(Step): def __init__(self, data_desc=None, final=False, fbk_timeout=None, fbk_mode=None, set_periodic=None, clear_periodic=None, step_desc=None, - do_before_data_processing=None): - Step.__init__(self, final=True, do_before_data_processing=do_before_data_processing) + do_before_data_processing=None, valid=True, vtg_ids=None): + Step.__init__(self, final=True, do_before_data_processing=do_before_data_processing, + valid=valid, vtg_ids=vtg_ids) class NoDataStep(Step): def __init__(self, data_desc=None, final=False, fbk_timeout=None, fbk_mode=None, set_periodic=None, clear_periodic=None, step_desc=None, - do_before_data_processing=None): + do_before_data_processing=None, valid=True, vtg_ids=None): Step.__init__(self, data_desc=Data(''), final=final, fbk_timeout=fbk_timeout, fbk_mode=fbk_mode, set_periodic=set_periodic, clear_periodic=clear_periodic, - step_desc=step_desc, do_before_data_processing=do_before_data_processing) + step_desc=step_desc, do_before_data_processing=do_before_data_processing, + valid=valid, vtg_ids=vtg_ids) self.make_blocked() def make_free(self): @@ -599,10 +642,13 @@ def __copy__(self): class ScenarioEnv(object): + knowledge_source = None + def __init__(self): self._dm = None self._target = None self._scenario = None + # self._knowledge_source = None @property def dm(self): @@ -628,11 +674,20 @@ def scenario(self): def scenario(self, val): self._scenario = val + # @property + # def knowledge_source(self): + # return self._knowledge_source + # + # @knowledge_source.setter + # def knowledge_source(self, val): + # self._knowledge_source = val + def __copy__(self): new_env = type(self)() new_env.__dict__.update(self.__dict__) new_env._target = None new_env._scenario = None + # new_env._knowledge_source = None return new_env @@ -675,6 +730,14 @@ def set_data_model(self, dm): def set_target(self, target): self._env.target = target + # @property + # def knowledge_source(self): + # return self._env.knowledge_source + # + # @knowledge_source.setter + # def knowledge_source(self, val): + # self._env.knowledge_source = val + def _graph_setup(self, init_step, steps, transitions): for tr in init_step.transitions: transitions.append(tr) @@ -688,6 +751,7 @@ def _init_main_properties(self): assert self._anchor is not None self._steps = [] self._transitions = [] + self._steps.append(self._anchor) self._graph_setup(self._anchor, self._steps, self._transitions) def _init_reinit_seq_properties(self): diff --git a/framework/tactics_helpers.py b/framework/tactics_helpers.py index 36d7205..7cd06ea 100644 --- a/framework/tactics_helpers.py +++ b/framework/tactics_helpers.py @@ -47,6 +47,7 @@ def __init__(self): self.disruptor_clones = {} self.generator_clones = {} self._fmkops = None + # self._knowledge_source = None def set_exportable_fmk_ops(self, fmkops): self._fmkops = fmkops @@ -57,6 +58,20 @@ def set_exportable_fmk_ops(self, fmkops): for name, attrs in self.get_disruptors_list(dtype).items(): attrs['obj'].set_exportable_fmk_ops(fmkops) + # @property + # def knowledge_source(self): + # return self._knowledge_source + # + # @knowledge_source.setter + # def knowledge_source(self, val): + # self._knowledge_source = val + # for dtype in self.generator_types: + # for name, attrs in self.get_generators_list(dtype).items(): + # attrs['obj'].knowledge_source = val + # for dtype in self.disruptor_types: + # for name, attrs in self.get_disruptors_list(dtype).items(): + # attrs['obj'].knowledge_source = val + def register_scenarios(self, *scenarios): for sc in scenarios: dyn_generator_from_scenario.scenario = sc @@ -94,18 +109,17 @@ def __register_new_data_maker(self, dict_var, name, obj, weight, dmaker_type, va dict_var[dmaker_type][XT_VALID_CLS_LIST_K][name] = \ dict_var[dmaker_type][XT_NAME_LIST_K][name] + if self._fmkops is not None: + obj.set_exportable_fmk_ops(self._fmkops) + # obj.knowledge_source = self.knowledge_source def register_new_disruptor(self, name, obj, weight, dmaker_type, valid=False): self.__register_new_data_maker(self.disruptors, name, obj, weight, dmaker_type, valid) - if self._fmkops: - obj.set_exportable_fmk_ops(self._fmkops) def register_new_generator(self, name, obj, weight, dmaker_type, valid=False): self.__register_new_data_maker(self.generators, name, obj, weight, dmaker_type, valid) - if self._fmkops: - obj.set_exportable_fmk_ops(self._fmkops) def __clone_dmaker(self, dmaker, dmaker_clones, dmaker_type, new_dmaker_type, dmaker_name=None, register_func=None): if dmaker_type not in dmaker: @@ -377,55 +391,59 @@ class UI(object): Once initialized, attributes cannot be modified ''' def __init__(self, **kwargs): - self.inputs = {} + self._inputs = {} for k, v in kwargs.items(): - self.inputs[k] = v + self._inputs[k] = v + + # for python2 compatibility + def __nonzero__(self): + return bool(self._inputs) + + # for python3 compatibility + def __bool__(self): + return bool(self._inputs) + + def get_inputs(self): + return self._inputs def is_attrs_defined(self, *names): for n in names: - if n not in self.inputs: + if n not in self._inputs: return False return True def set_user_inputs(self, user_inputs): assert isinstance(user_inputs, dict) - self.inputs = user_inputs + self._inputs = user_inputs def check_conformity(self, valid_args): - for arg in self.inputs: + for arg in self._inputs: if arg not in valid_args: return False, arg return True, None def __getattr__(self, name): - if name in self.inputs: - return self.inputs[name] + if name in self._inputs: + return self._inputs[name] else: return None def __str__(self): - if self.inputs: + if self._inputs: ui = '[' - for k, v in self.inputs.items(): + for k, v in self._inputs.items(): ui += "{:s}={!r},".format(k, v) return ui[:-1]+']' else: return '[ ]' -def _user_input_conformity(self, user_input, _gen_args_desc, _args_desc): +def _user_input_conformity(self, user_input, _args_desc): if not user_input: return True - generic_ui = user_input.get_generic() - specific_ui = user_input.get_specific() - if _gen_args_desc and generic_ui is not None: - ok, guilty = generic_ui.check_conformity(_gen_args_desc.keys()) - if not ok: - print("\n*** Unknown parameter: '{:s}'".format(guilty)) - return False - if _args_desc and specific_ui is not None: - ok, guilty = specific_ui.check_conformity(_args_desc.keys()) + if _args_desc: + ok, guilty = user_input.check_conformity(_args_desc.keys()) if not ok: print("\n*** Unknown parameter: '{:s}'".format(guilty)) return False @@ -433,41 +451,16 @@ def _user_input_conformity(self, user_input, _gen_args_desc, _args_desc): return True -def _handle_user_inputs(dmaker, ui): - generic_ui = ui.get_generic() - specific_ui = ui.get_specific() - if generic_ui is None: - for k, v in dmaker._gen_args_desc.items(): - desc, default, arg_type = v - setattr(dmaker, k, default) - else: - for k, v in dmaker._gen_args_desc.items(): - desc, default, arg_type = v - ui_val = getattr(generic_ui, k) - if isinstance(arg_type, tuple): - assert(type(ui_val) in arg_type or ui_val is None) - elif isinstance(arg_type, type): - assert(type(ui_val) == arg_type or ui_val is None) - else: - raise ValueError - if ui_val is None: - setattr(dmaker, k, default) - else: - setattr(dmaker, k, ui_val) - - if dmaker._gen_args_desc \ - and issubclass(dmaker.__class__, (Disruptor, StatefulDisruptor)) \ - and dmaker._gen_args_desc == GENERIC_ARGS: - modelwalker_inputs_handling_helper(dmaker, generic_ui) +def _handle_user_inputs(dmaker, user_input): - if specific_ui is None: + if user_input is None: for k, v in dmaker._args_desc.items(): desc, default, arg_type = v setattr(dmaker, k, default) else: for k, v in dmaker._args_desc.items(): desc, default, arg_type = v - ui_val = getattr(specific_ui, k) + ui_val = getattr(user_input, k) if isinstance(arg_type, tuple): assert(type(ui_val) in arg_type or ui_val is None) elif isinstance(arg_type, type): @@ -479,42 +472,16 @@ def _handle_user_inputs(dmaker, ui): else: setattr(dmaker, k, ui_val) + if isinstance(dmaker, DataMaker) and dmaker.modelwalker_user: + modelwalker_inputs_handling_helper(dmaker) + def _restore_dmaker_internals(dmaker): - for k, v in dmaker._gen_args_desc.items(): - desc, default, arg_type = v - setattr(dmaker, k, default) for k, v in dmaker._args_desc.items(): desc, default, arg_type = v setattr(dmaker, k, default) -class UserInputContainer(object): - - def __init__(self, generic=None, specific=None): - self._generic_input = generic - self._specific_input = specific - - # for python2 compatibility - def __nonzero__(self): - return self._generic_input is not None or self._specific_input is not None - - # for python3 compatibility - def __bool__(self): - return self._generic_input is not None or self._specific_input is not None - - def get_generic(self): - return self._generic_input - - def get_specific(self): - return self._specific_input - - def __str__(self): - return "G="+str(self._generic_input)+", S="+str(self._specific_input) - - def __repr__(self): - return str(self) - ################################ # ModelWalker Helper Functions # ################################ @@ -528,7 +495,7 @@ def __repr__(self): 'big data it can be usefull to it to False)', True, bool) } -def modelwalker_inputs_handling_helper(dmaker, user_generic_input): +def modelwalker_inputs_handling_helper(dmaker): assert(dmaker.runs_per_node > 0 or dmaker.runs_per_node == -1) if dmaker.runs_per_node == -1: @@ -547,20 +514,33 @@ class DataMakerAttr: SetupRequired = 4 NeedSeed = 5 -class Generator(object): +class DataMaker(object): + knowledge_source = None + _modelwalker_user = False + _args_desc = None + + def __init__(self): + self._fmkops = None + # self._knowledge_source = None + + def set_exportable_fmk_ops(self, fmkops): + self._fmkops = fmkops + + @property + def modelwalker_user(self): + return self._modelwalker_user + +class Generator(DataMaker): produced_seed = None def __init__(self): + DataMaker.__init__(self) self.__attrs = { DataMakerAttr.Active: True, DataMakerAttr.Controller: False, DataMakerAttr.HandOver: False, DataMakerAttr.SetupRequired: True } - self._fmkops = None - - def set_exportable_fmk_ops(self, fmkops): - self._fmkops = fmkops def set_attr(self, name): if name not in self.__attrs: @@ -580,9 +560,8 @@ def is_attr_set(self, name): return self.__attrs[name] def _setup(self, dm, user_input): - # sys.stdout.write("\n__ setup generator '%s' __" % self.__class__.__name__) self.clear_attr(DataMakerAttr.SetupRequired) - if not _user_input_conformity(self, user_input, self._gen_args_desc, self._args_desc): + if not _user_input_conformity(self, user_input, self._args_desc): return False _handle_user_inputs(self, user_input) @@ -598,14 +577,11 @@ def _setup(self, dm, user_input): return ok def _cleanup(self): - # sys.stdout.write("\n__ cleanup generator '%s' __" % self.__class__.__name__) self.set_attr(DataMakerAttr.Active) self.set_attr(DataMakerAttr.SetupRequired) self.cleanup(self._fmkops) - def need_reset(self): - # sys.stdout.write("\n__ generator need reset '%s' __" % self.__class__.__name__) self.set_attr(DataMakerAttr.SetupRequired) self.cleanup(self._fmkops) @@ -630,7 +606,6 @@ class dyn_generator(type): data_id = '' def __init__(cls, name, bases, attrs): - attrs['_gen_args_desc'] = DynGenerator._gen_args_desc attrs['_args_desc'] = DynGenerator._args_desc type.__init__(cls, name, bases, attrs) cls.data_id = dyn_generator.data_id @@ -638,12 +613,11 @@ def __init__(cls, name, bases, attrs): class DynGenerator(Generator): data_id = '' - _gen_args_desc = { + _args_desc = { 'finite': ('make the data model finite', False, bool), 'determinist': ('make the data model determinist', False, bool), 'random': ('make the data model random', False, bool) } - _args_desc = {} def setup(self, dm, user_input): if self.determinist or self.random: @@ -667,14 +641,13 @@ def generate_data(self, dm, monitor, target): class dyn_generator_from_scenario(type): scenario = None def __init__(cls, name, bases, attrs): - attrs['_gen_args_desc'] = DynGeneratorFromScenario._gen_args_desc attrs['_args_desc'] = DynGeneratorFromScenario._args_desc type.__init__(cls, name, bases, attrs) cls.scenario = dyn_generator_from_scenario.scenario class DynGeneratorFromScenario(Generator): scenario = None - _gen_args_desc = collections.OrderedDict([ + _args_desc = collections.OrderedDict([ ('graph', ('Display the scenario and highlight the current step each time the generator ' 'is called.', False, bool)), ('graph_format', ('Format to be used for displaying the scenario (e.g., xdot, pdf, png).', @@ -700,7 +673,6 @@ class DynGeneratorFromScenario(Generator): "the generator begin with the Nth corrupted scenario (where N is provided " "through this parameter).", 0, int)) ]) - _args_desc = {} @property def produced_seed(self): @@ -726,9 +698,10 @@ def _cleanup_walking_attrs(self): self.tr_selected_idx = -1 def setup(self, dm, user_input): - if not _user_input_conformity(self, user_input, self._gen_args_desc, self._args_desc): + if not _user_input_conformity(self, user_input, self._args_desc): return False self.__class__.scenario.set_data_model(dm) + # self.__class__.scenario.knowledge_source = self.knowledge_source self.scenario = copy.copy(self.__class__.scenario) assert (self.data_fuzz and not (self.cond_fuzz or self.ignore_timing)) or not self.data_fuzz @@ -799,7 +772,7 @@ def _alter_data_step(self): or (isinstance(data_desc[0], Data) and data_desc[0].content is not None): dp = sc.DataProcess(process=['tTYPE#{:d}'.format(self._step_num)], seed=data_desc[0], auto_regen=True) - dp.append_new_process([('tSTRUCT#{:d}'.format(self._step_num), UI(init=1), UI(deep=True))]) + dp.append_new_process([('tSTRUCT#{:d}'.format(self._step_num), UI(init=1, deep=True))]) data_desc[0] = dp step.data_desc = data_desc elif isinstance(data_desc[0], sc.DataProcess): @@ -807,7 +780,7 @@ def _alter_data_step(self): proc2 = copy.copy(data_desc[0].process) proc.append('tTYPE#{:d}'.format(self._step_num)) data_desc[0].process = proc - proc2.append(('tSTRUCT#{:d}'.format(self._step_num), UI(init=1), UI(deep=True))) + proc2.append(('tSTRUCT#{:d}'.format(self._step_num), UI(init=1, deep=True))) data_desc[0].append_new_process(proc2) data_desc[0].auto_regen = True elif isinstance(data_desc[0], Data): @@ -934,6 +907,7 @@ def generate_data(self, dm, monitor, target): data = self.step.get_data() data.origin = self.scenario data.cleanup_all_callbacks() + data.altered = not self.step.valid if self.cond_fuzz or self.ignore_timing or self.data_fuzz: data.add_info("Current fuzzed step: '{:s}'" @@ -943,8 +917,12 @@ def generate_data(self, dm, monitor, target): data.register_callback(self._callback_dispatcher_before_sending_step2, hook=HOOK.before_sending_step2) data.register_callback(self._callback_dispatcher_after_sending, hook=HOOK.after_sending) data.register_callback(self._callback_dispatcher_after_fbk, hook=HOOK.after_fbk) + + data.scenario_dependence = self.scenario.name + return data + def _callback_cleanup_periodic(self): cbkops = CallBackOps() for periodic_id in self.scenario.periodic_to_clear: @@ -984,16 +962,17 @@ def _callback_dispatcher_before_sending_step1(self): cbkops = CallBackOps() if self.step.has_dataprocess(): cbkops.add_operation(CallBackOps.Replace_Data, - param=self.step.data_desc) + param=(self.step.data_desc, self.step.vtg_ids_list)) return cbkops def _callback_dispatcher_before_sending_step2(self): # Callback called after any data have been processed but not sent yet self.step.do_before_sending() + # We add again the operation CallBackOps.Replace_Data, because the step contents could have changed cbkops = CallBackOps() cbkops.add_operation(CallBackOps.Replace_Data, - param=self.step.data_desc) + param=(self.step.data_desc, self.step.vtg_ids_list)) return cbkops def _callback_dispatcher_after_sending(self): @@ -1024,19 +1003,16 @@ def _callback_dispatcher_after_fbk(self, fbk): return cbkops -class Disruptor(object): +class Disruptor(DataMaker): def __init__(self): + DataMaker.__init__(self) self.__attrs = { DataMakerAttr.Active: True, DataMakerAttr.Controller: False, DataMakerAttr.HandOver: False, DataMakerAttr.SetupRequired: True } - self._fmkops = None - - def set_exportable_fmk_ops(self, fmkops): - self._fmkops = fmkops def disrupt_data(self, dm, target, prev_data): raise NotImplementedError @@ -1073,7 +1049,7 @@ def is_attr_set(self, name): def _setup(self, dm, user_input): # sys.stdout.write("\n__ setup disruptor '%s' __" % self.__class__.__name__) self.clear_attr(DataMakerAttr.SetupRequired) - if not _user_input_conformity(self, user_input, self._gen_args_desc, self._args_desc): + if not _user_input_conformity(self, user_input, self._args_desc): return False _handle_user_inputs(self, user_input) @@ -1097,9 +1073,10 @@ def _cleanup(self): -class StatefulDisruptor(object): +class StatefulDisruptor(DataMaker): def __init__(self): + DataMaker.__init__(self) self.__attrs = { DataMakerAttr.Active: True, DataMakerAttr.Controller: False, @@ -1107,10 +1084,6 @@ def __init__(self): DataMakerAttr.SetupRequired: True, DataMakerAttr.NeedSeed: True } - self._fmkops = None - - def set_exportable_fmk_ops(self, fmkops): - self._fmkops = fmkops def set_seed(self, prev_data): raise NotImplementedError @@ -1160,7 +1133,7 @@ def is_attr_set(self, name): def _setup(self, dm, user_input): # sys.stdout.write("\n__ setup disruptor '%s' __" % self.__class__.__name__) self.clear_attr(DataMakerAttr.SetupRequired) - if not _user_input_conformity(self, user_input, self._gen_args_desc, self._args_desc): + if not _user_input_conformity(self, user_input, self._args_desc): return False _handle_user_inputs(self, user_input) @@ -1189,14 +1162,17 @@ def _set_seed(self, prev_data): return ret -def disruptor(st, dtype, weight=1, valid=False, gen_args={}, args={}): +def disruptor(st, dtype, weight=1, valid=False, args=None, modelwalker_user=False): def internal_func(disruptor_cls): - disruptor_cls._gen_args_desc = gen_args - disruptor_cls._args_desc = args - # check conflict between gen_args & args - for k in gen_args: - if k in args.keys(): - raise ValueError("Specific parameter '{:s}' is in conflict with a generic parameter!".format(k)) + disruptor_cls._modelwalker_user = modelwalker_user + if modelwalker_user: + if set(GENERIC_ARGS.keys()).intersection(set(args.keys())): + raise ValueError('At least one parameter is in conflict with a built-in parameter') + disruptor_cls._args_desc = copy.copy(GENERIC_ARGS) + if args: + disruptor_cls._args_desc.update(args) + else: + disruptor_cls._args_desc = {} if args is None else args # register an object of this class disruptor = disruptor_cls() if issubclass(disruptor_cls, StatefulDisruptor): @@ -1208,14 +1184,17 @@ def internal_func(disruptor_cls): return internal_func -def generator(st, gtype, weight=1, valid=False, gen_args={}, args={}): +def generator(st, gtype, weight=1, valid=False, args=None, modelwalker_user=False): def internal_func(generator_cls): - generator_cls._gen_args_desc = gen_args - generator_cls._args_desc = args - # check conflict between gen_args & args - for k in gen_args: - if k in args.keys(): - raise ValueError("Specific parameter '{:s}' is in conflict with a generic parameter!".format(k)) + generator_cls._modelwalker_user = modelwalker_user + if modelwalker_user: + if set(GENERIC_ARGS.keys()).intersection(set(args.keys())): + raise ValueError('At least one parameter is in conflict with a built-in parameter') + generator_cls._args_desc = copy.copy(GENERIC_ARGS) + if args: + generator_cls._args_desc.update(args) + else: + generator_cls._args_desc = {} if args is None else args # register an object of this class gen = generator_cls() st.register_new_generator(gen.__class__.__name__, gen, weight, gtype, valid) diff --git a/framework/target_helpers.py b/framework/target_helpers.py index 521c626..7b94824 100644 --- a/framework/target_helpers.py +++ b/framework/target_helpers.py @@ -21,12 +21,10 @@ # ################################################################################ -import collections import datetime import threading from framework.data import Data -from framework.global_resources import * from libs.external_modules import * class TargetStuck(Exception): pass @@ -36,6 +34,9 @@ class Target(object): Class abstracting the target we interact with. ''' feedback_timeout = None + sending_delay = 0 + + # tg_id = None # this is set by FmkPlumbing FBK_WAIT_FULL_TIME = 1 fbk_wait_full_time_slot_msg = 'Wait for the full time slot allocated for feedback retrieval' @@ -49,18 +50,26 @@ class Target(object): _probes = None _send_data_lock = threading.Lock() + _altered_data_queued = None + + _pending_data = None + def set_logger(self, logger): self._logger = logger def set_data_model(self, dm): self.current_dm = dm - def _start(self): - self._logger.print_console('*** Target initialization ***\n', nl_before=False, rgb=Color.COMPONENT_START) + def _start(self, target_desc, tg_id): + self._logger.print_console('*** Target initialization: ({:d}) {!s} ***\n'.format(tg_id, target_desc), + nl_before=False, rgb=Color.COMPONENT_START) + self._pending_data = [] return self.start() - def _stop(self): - self._logger.print_console('*** Target cleanup procedure ***\n', nl_before=False, rgb=Color.COMPONENT_STOP) + def _stop(self, target_desc, tg_id): + self._logger.print_console('*** Target cleanup procedure for ({:d}) {!s} ***\n'.format(tg_id, target_desc), + nl_before=False, rgb=Color.COMPONENT_STOP) + self._pending_data = None return self.stop() def start(self): @@ -155,7 +164,7 @@ def recover_target(self): def get_feedback(self): ''' - If overloaded, should return a TargetFeedback object. + If overloaded, should return a FeedbackCollector object. ''' return None @@ -202,8 +211,6 @@ def set_feedback_mode(self, mode): def fbk_wait_full_time_slot_mode(self): return self._feedback_mode == Target.FBK_WAIT_FULL_TIME - sending_delay = 0 - def set_sending_delay(self, sending_delay): ''' Set the sending delay. @@ -215,8 +222,30 @@ def set_sending_delay(self, sending_delay): assert sending_delay >= 0 self.sending_delay = sending_delay + def __str__(self): + return self.__class__.__name__ + ' [' + self.get_description() + ']' + def get_description(self): - return None + return 'ID: ' + str(id(self))[-6:] + + def add_pending_data(self, data): + with self._send_data_lock: + if isinstance(data, list): + self._pending_data += data + else: + self._pending_data.append(data) + + def send_pending_data(self, from_fmk=False): + with self._send_data_lock: + data_list = self._pending_data + self._pending_data = [] + + if len(data_list) == 1: + self.send_data_sync(data_list[0], from_fmk=from_fmk) + elif len(data_list) > 1: + self.send_multiple_data_sync(data_list, from_fmk=from_fmk) + else: + raise ValueError('No pending data') def send_data_sync(self, data, from_fmk=False): ''' @@ -228,6 +257,7 @@ def send_data_sync(self, data, from_fmk=False): emits the message by itself. ''' with self._send_data_lock: + self._altered_data_queued = data.altered self.send_data(data, from_fmk=from_fmk) def send_multiple_data_sync(self, data_list, from_fmk=False): @@ -236,6 +266,7 @@ def send_multiple_data_sync(self, data_list, from_fmk=False): with the framework. ''' with self._send_data_lock: + self._altered_data_queued = data_list[0].altered self.send_multiple_data(data_list, from_fmk=from_fmk) def add_probe(self, probe): @@ -246,6 +277,9 @@ def add_probe(self, probe): def remove_probes(self): self._probes = None + def is_processed_data_altered(self): + return self._altered_data_queued + @property def probes(self): return self._probes if self._probes is not None else [] @@ -270,72 +304,10 @@ def send_multiple_data(self, data_list, from_fmk=False): self._sending_time = datetime.datetime.now() def is_target_ready_for_new_data(self): - if self._feedback_enabled and self._sending_time is not None: + if self._feedback_enabled and self.feedback_timeout is not None and \ + self._sending_time is not None: return (datetime.datetime.now() - self._sending_time).total_seconds() > self.feedback_timeout else: return True -class TargetFeedback(object): - fbk_lock = threading.Lock() - - def __init__(self): - self.cleanup() - self._feedback_collector = collections.OrderedDict() - self._feedback_collector_tstamped = collections.OrderedDict() - self._tstamped_bstring = None - # self.set_bytes(bstring) - - def add_fbk_from(self, ref, fbk, status=0): - now = datetime.datetime.now() - with self.fbk_lock: - if ref not in self._feedback_collector: - self._feedback_collector[ref] = {} - self._feedback_collector[ref]['data'] = [] - self._feedback_collector[ref]['status'] = 0 - self._feedback_collector_tstamped[ref] = [] - if fbk.strip() not in self._feedback_collector[ref]: - self._feedback_collector[ref]['data'].append(fbk) - self._feedback_collector[ref]['status'] = status - self._feedback_collector_tstamped[ref].append(now) - - def has_fbk_collector(self): - return len(self._feedback_collector) > 0 - - def __iter__(self): - with self.fbk_lock: - fbk_collector = copy.copy(self._feedback_collector) - fbk_collector_ts = copy.copy(self._feedback_collector_tstamped) - for ref, fbk in fbk_collector.items(): - yield ref, fbk['data'], fbk['status'], fbk_collector_ts[ref] - - def iter_and_cleanup_collector(self): - with self.fbk_lock: - fbk_collector = self._feedback_collector - fbk_collector_ts = self._feedback_collector_tstamped - self._feedback_collector = collections.OrderedDict() - self._feedback_collector_tstamped = collections.OrderedDict() - for ref, fbk in fbk_collector.items(): - yield ref, fbk['data'], fbk['status'], fbk_collector_ts[ref] - - def set_error_code(self, err_code): - self._err_code = err_code - - def get_error_code(self): - return self._err_code - - def set_bytes(self, bstring): - now = datetime.datetime.now() - self._tstamped_bstring = (bstring, now) - - def get_bytes(self): - return None if self._tstamped_bstring is None else self._tstamped_bstring[0] - - def get_timestamp(self): - return None if self._tstamped_bstring is None else self._tstamped_bstring[1] - - def cleanup(self): - # fbk_collector cleanup is done during consumption to avoid loss of feedback in - # multi-threading context - self._tstamped_bstring = None - self.set_error_code(0) diff --git a/framework/targets/debug.py b/framework/targets/debug.py index a34bbb9..46dac5e 100644 --- a/framework/targets/debug.py +++ b/framework/targets/debug.py @@ -22,28 +22,39 @@ ################################################################################ import random +import datetime +import time from framework.target_helpers import Target +from framework.basic_primitives import rand_string class TestTarget(Target): _feedback_mode = None - supported_feedback_mode = [] + supported_feedback_mode = [Target.FBK_WAIT_UNTIL_RECV] + _last_ack_date = None - def __init__(self, recover_ratio=100): + def __init__(self, recover_ratio=100, fbk_samples=None): Target.__init__(self) self._cpt = None self._recover_ratio = recover_ratio + self._fbk_samples = fbk_samples def start(self): self._cpt = 0 return True def send_data(self, data, from_fmk=False): - pass + self._last_ack_date = datetime.datetime.now() + datetime.timedelta(microseconds=random.randint(20, 40)) + time.sleep(0.001) + fbk_content = random.choice(self._fbk_samples) if self._fbk_samples else rand_string(size=10) + self._logger.collect_feedback(content=fbk_content, status_code=random.randint(-3, 3)) def send_multiple_data(self, data_list, from_fmk=False): - pass + self._last_ack_date = datetime.datetime.now() + datetime.timedelta(microseconds=random.randint(20, 40)) + time.sleep(0.001) + fbk_content = random.choice(self._fbk_samples) if self._fbk_samples else rand_string(size=20) + self._logger.collect_feedback(content=fbk_content, status_code=random.randint(-3, 3)) def is_target_ready_for_new_data(self): self._cpt += 1 @@ -57,4 +68,7 @@ def recover_target(self): if random.randint(1, 100) > (100 - self._recover_ratio): return True else: - return False \ No newline at end of file + return False + + def get_last_target_ack_date(self): + return self._last_ack_date diff --git a/framework/targets/local.py b/framework/targets/local.py index 0e9b0e8..93bee19 100644 --- a/framework/targets/local.py +++ b/framework/targets/local.py @@ -29,45 +29,56 @@ import subprocess from framework.global_resources import workspace_folder -from framework.target_helpers import Target, TargetFeedback +from framework.target_helpers import Target +from framework.knowledge.feedback_collector import FeedbackCollector + class LocalTarget(Target): _feedback_mode = Target.FBK_WAIT_UNTIL_RECV supported_feedback_mode = [Target.FBK_WAIT_UNTIL_RECV] - def __init__(self, tmpfile_ext, target_path=None): + def __init__(self, target_path=None, pre_args='', post_args='', + tmpfile_ext='.bin', send_via_stdin=False, send_via_cmdline=False): Target.__init__(self) - self.__suffix = '{:0>12d}'.format(random.randint(2**16, 2**32)) - self.__app = None - self.__pre_args = None - self.__post_args = None + self._suffix = '{:0>12d}'.format(random.randint(2 ** 16, 2 ** 32)) + self._app = None + self._pre_args = pre_args + self._post_args = post_args + self._send_via_stdin = send_via_stdin + self._send_via_cmdline = send_via_cmdline self._data_sent = None self._feedback_computed = None - self.__feedback = TargetFeedback() + self._feedback = FeedbackCollector() self.set_target_path(target_path) self.set_tmp_file_extension(tmpfile_ext) + def get_description(self): + pre_args = self._pre_args + post_args = self._post_args + args = ', Args: ' + pre_args + post_args if pre_args or post_args else '' + return 'Program: ' + self._target_path + args + def set_tmp_file_extension(self, tmpfile_ext): self._tmpfile_ext = tmpfile_ext def set_target_path(self, target_path): - self.__target_path = target_path + self._target_path = target_path def get_target_path(self): - return self.__target_path + return self._target_path def set_pre_args(self, pre_args): - self.__pre_args = pre_args + self._pre_args = pre_args def get_pre_args(self): - return self.__pre_args + return self._pre_args def set_post_args(self, post_args): - self.__post_args = post_args + self._post_args = post_args def get_post_args(self): - return self.__post_args + return self._post_args def initialize(self): ''' @@ -82,7 +93,7 @@ def terminate(self): return True def start(self): - if not self.__target_path: + if not self._target_path: print('/!\\ ERROR /!\\: the LocalTarget path has not been set') return False @@ -99,66 +110,78 @@ def _before_sending_data(self): def send_data(self, data, from_fmk=False): self._before_sending_data() data = data.to_bytes() - wkspace = workspace_folder - - name = os.path.join(wkspace, 'fuzz_test_' + self.__suffix + self._tmpfile_ext) - with open(name, 'wb') as f: - f.write(data) - - if self.__pre_args is not None and self.__post_args is not None: - cmd = [self.__target_path] + self.__pre_args.split() + [name] + self.__post_args.split() - elif self.__pre_args is not None: - cmd = [self.__target_path] + self.__pre_args.split() + [name] - elif self.__post_args is not None: - cmd = [self.__target_path, name] + self.__post_args.split() + + if self._send_via_stdin: + name = '' + elif self._send_via_cmdline: + name = data else: - cmd = [self.__target_path, name] + name = os.path.join(workspace_folder, 'fuzz_test_' + self._suffix + self._tmpfile_ext) + with open(name, 'wb') as f: + f.write(data) + + if self._pre_args is not None and self._post_args is not None: + cmd = [self._target_path] + self._pre_args.split() + [name] + self._post_args.split() + elif self._pre_args is not None: + cmd = [self._target_path] + self._pre_args.split() + [name] + elif self._post_args is not None: + cmd = [self._target_path, name] + self._post_args.split() + else: + cmd = [self._target_path, name] + + stdin_arg = subprocess.PIPE if self._send_via_stdin else None + self._app = subprocess.Popen(args=cmd, stdin=stdin_arg, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + if self._send_via_stdin: + with self._app.stdin as f: + f.write(data) - self.__app = subprocess.Popen(args=cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if not self._send_via_stdin and not self._send_via_cmdline: + fl = fcntl.fcntl(self._app.stderr, fcntl.F_GETFL) + fcntl.fcntl(self._app.stderr, fcntl.F_SETFL, fl | os.O_NONBLOCK) - fl = fcntl.fcntl(self.__app.stderr, fcntl.F_GETFL) - fcntl.fcntl(self.__app.stderr, fcntl.F_SETFL, fl | os.O_NONBLOCK) + fl = fcntl.fcntl(self._app.stdout, fcntl.F_GETFL) + fcntl.fcntl(self._app.stdout, fcntl.F_SETFL, fl | os.O_NONBLOCK) - fl = fcntl.fcntl(self.__app.stdout, fcntl.F_GETFL) - fcntl.fcntl(self.__app.stdout, fcntl.F_SETFL, fl | os.O_NONBLOCK) self._data_sent = True def cleanup(self): - if self.__app is None: + if self._app is None: return try: - os.kill(self.__app.pid, signal.SIGTERM) + os.kill(self._app.pid, signal.SIGTERM) except: - print("\n*** WARNING: cannot kill application with PID {:d}".format(self.__app.pid)) + print("\n*** WARNING: cannot kill application with PID {:d}".format(self._app.pid)) finally: self._data_sent = False def get_feedback(self, timeout=0.2): timeout = self.feedback_timeout if timeout is None else timeout if self._feedback_computed: - return self.__feedback + return self._feedback else: self._feedback_computed = True err_detected = False - if self.__app is None and self._data_sent: + if self._app is None and self._data_sent: err_detected = True - self.__feedback.add_fbk_from("LocalTarget", "Application has terminated (crash?)", - status=-3) - return self.__feedback - elif self.__app is None: - return self.__feedback + self._feedback.add_fbk_from("LocalTarget", "Application has terminated (crash?)", + status=-3) + return self._feedback + elif self._app is None: + return self._feedback - exit_status = self.__app.poll() + exit_status = self._app.poll() if exit_status is not None and exit_status < 0: err_detected = True - self.__feedback.add_fbk_from("Application[{:d}]".format(self.__app.pid), + self._feedback.add_fbk_from("Application[{:d}]".format(self._app.pid), "Negative return status ({:d})".format(exit_status), - status=exit_status) + status=exit_status) - ret = select.select([self.__app.stdout, self.__app.stderr], [], [], timeout) + ret = select.select([self._app.stdout, self._app.stderr], [], [], timeout) if ret[0]: byte_string = b'' for fd in ret[0][:-1]: @@ -166,16 +189,16 @@ def get_feedback(self, timeout=0.2): if b'error' in byte_string or b'invalid' in byte_string: err_detected = True - self.__feedback.add_fbk_from("LocalTarget[stdout]", + self._feedback.add_fbk_from("LocalTarget[stdout]", "Application outputs errors on stdout", - status=-1) + status=-1) stderr_msg = ret[0][-1].read() if stderr_msg: err_detected = True - self.__feedback.add_fbk_from("LocalTarget[stderr]", + self._feedback.add_fbk_from("LocalTarget[stderr]", "Application outputs on stderr", - status=-2) + status=-2) byte_string += stderr_msg else: byte_string = byte_string[:-2] # remove '\n\n' @@ -184,7 +207,7 @@ def get_feedback(self, timeout=0.2): byte_string = b'' if err_detected: - self.__feedback.set_error_code(-1) - self.__feedback.set_bytes(byte_string) + self._feedback.set_error_code(-1) + self._feedback.set_bytes(byte_string) - return self.__feedback \ No newline at end of file + return self._feedback \ No newline at end of file diff --git a/framework/targets/network.py b/framework/targets/network.py index dcd8a0a..a3843e2 100644 --- a/framework/targets/network.py +++ b/framework/targets/network.py @@ -36,7 +36,9 @@ from framework.data import Data from framework.node import Node, NodeSemanticsCriteria -from framework.target_helpers import Target, TargetFeedback, TargetStuck +from framework.target_helpers import Target, TargetStuck +from framework.knowledge.feedback_collector import FeedbackCollector + class NetworkTarget(Target): '''Generic target class for interacting with a network resource. Can @@ -44,6 +46,8 @@ class NetworkTarget(Target): fit your needs. ''' + General_Info_ID = 'General Information' + UNKNOWN_SEMANTIC = "Unknown Semantic" CHUNK_SZ = 2048 _INTERNALS_ID = 'NetworkTarget()' @@ -53,7 +57,7 @@ class NetworkTarget(Target): def __init__(self, host='localhost', port=12345, socket_type=(socket.AF_INET, socket.SOCK_STREAM), data_semantics=UNKNOWN_SEMANTIC, server_mode=False, target_address=None, wait_for_client=True, - hold_connection=False, + hold_connection=False, keep_first_client=True, mac_src=None, mac_dst=None): """ Args: @@ -87,6 +91,11 @@ def __init__(self, host='localhost', port=12345, socket_type=(socket.AF_INET, so hold_connection (bool): If `True`, we will maintain the connection while sending data to the real target. Otherwise, after each data emission, we close the related socket. + keep_first_client (bool): Used only in server mode (`server_mode` is `True`) with `SOCK_STREAM` + socket type. If set to `True`, the first client that connects to the server will remain + the one used for data sending until the target is reloaded. Otherwise, last client + information are used. This is not supported for `SOCK_DGRAM` where the first client will + always be the one used for data sending. mac_src (bytes): Only in conjunction with raw socket. For each data sent through this interface, and if this data contain nodes with the semantic ``'mac_src'``, these nodes will be overwritten (through absorption) with this parameter. If nothing @@ -132,7 +141,7 @@ def get_mac_addr(ifname): self._mac_dst[(host, port)] = mac_dst self._server_mode_additional_info = {} - self._server_mode_additional_info[(host, port)] = (target_address, wait_for_client) + self._server_mode_additional_info[(host, port)] = (target_address, wait_for_client, keep_first_client) self._host = {} self._port = {} @@ -145,7 +154,7 @@ def get_mac_addr(ifname): self.sending_sockets = [] self.multiple_destination = False - self._feedback = TargetFeedback() + self._feedback = FeedbackCollector() self._fbk_handling_lock = threading.Lock() self.socket_desc_lock = threading.Lock() @@ -184,7 +193,8 @@ def _is_valid_socket_type(self, socket_type): def register_new_interface(self, host, port, socket_type, data_semantics, server_mode=False, target_address = None, wait_for_client=True, - hold_connection=False, mac_src=None, mac_dst=None): + hold_connection=False, keep_first_client=True, + mac_src=None, mac_dst=None): if not self._is_valid_socket_type(socket_type): raise ValueError("Unrecognized socket type") @@ -196,7 +206,7 @@ def register_new_interface(self, host, port, socket_type, data_semantics, server assert data_semantics not in self.known_semantics self.known_semantics.add(data_semantics) self.server_mode[(host,port)] = server_mode - self._server_mode_additional_info[(host, port)] = (target_address, wait_for_client) + self._server_mode_additional_info[(host, port)] = (target_address, wait_for_client, keep_first_client) self._default_fbk_id[(host, port)] = self._default_fbk_socket_id + ' - {:s}:{:d}'.format(host, port) self.hold_connection[(host, port)] = hold_connection if socket_type[1] == socket.SOCK_RAW: @@ -249,7 +259,7 @@ def add_additional_feedback_interface(self, host, port, assert(not str(fbk_id).startswith('Default Additional Feedback ID')) self._additional_fbk_desc[fbk_id] = (host, port, socket_type, fbk_id, fbk_length, server_mode) self.hold_connection[(host, port)] = True - self._server_mode_additional_info[(host, port)] = (None, None) + self._server_mode_additional_info[(host, port)] = (None, None, None) def _custom_data_handling_before_emission(self, data_list): '''To be overloaded if you want to perform some operation before @@ -292,6 +302,7 @@ def listen_to(self, host, port, ref_id, Used for collecting feedback from the target while it is already started. ''' self.hold_connection[(host, port)] = hold_connection + self._server_mode_additional_info[(host, port)] = (None, False, False) self._raw_listen_to(host, port, ref_id, socket_type, chk_size, wait_time=wait_time) self._dynamic_interfaces[(host, port)] = (-1, ref_id) @@ -430,7 +441,7 @@ def start(self): self._additional_fbk_ids = {} self._additional_fbk_lengths = {} self._dynamic_interfaces = {} - self._feedback_handled = None + self._feedback_handled = True self.feedback_thread_qty = 0 self.feedback_complete_cpt = 0 self._sending_id = 0 @@ -521,10 +532,11 @@ def send_multiple_data(self, data_list, from_fmk=False): port = self._port[key] socket_type = self._socket_type[key] server_mode = self.server_mode[(host, port)] - if self.hold_connection[(host, port)]: - # Collecting feedback makes sense only if we keep the socket (thus, 'hold_connection' - # has to be True) or if a data callback wait for feedback. - sending_list.append((None, host, port, socket_type, server_mode)) + sending_list.append((None, host, port, socket_type, server_mode)) + # if self.hold_connection[(host, port)]: + # # Collecting feedback makes sense only if we keep the socket (thus, 'hold_connection' + # # has to be True) or if a data callback wait for feedback. + # sending_list.append((None, host, port, socket_type, server_mode)) else: for data in data_list: host, port, socket_type, server_mode = self._get_net_info_from(data) @@ -552,8 +564,10 @@ def send_multiple_data(self, data_list, from_fmk=False): self._send_data(sockets, data_refs, self._sending_id, from_fmk) else: # this case exist when data are only sent through 'server_mode'-configured interfaces + # (because self._send_data() is called through self._handle_target_connection()) # or a connection error has occurred. pass + # self._feedback_handled = True if data_list is None: return @@ -708,7 +722,7 @@ def start_raw_server(serversocket, sending_event, notif_host_event): try: serversocket.bind((host, port)) except socket.error as serr: - print('\n*** ERROR(while binding socket|host={!s},port={:d}): {:s}'.format(host, port, str(serr))) + print('\n*** ERROR(while binding socket -- host={!s} port={:d}): {:s}'.format(host, port, str(serr))) return False serversocket.settimeout(self.sending_delay) @@ -736,10 +750,22 @@ def start_raw_server(serversocket, sending_event, notif_host_event): # For SOCK_STREAM def _server_main(self, serversocket, host, port, func): + _first_client = {} while not self.stop_event.is_set(): + _, _, keep_first_client = self._server_mode_additional_info[(host, port)] try: - # accept connections from outside - (clientsocket, address) = serversocket.accept() + fc_addr = _first_client.get((host,port), None) + clientsocket, address = serversocket.accept() + if keep_first_client and fc_addr is not None: + continue + elif keep_first_client: + _first_client[(host, port)] = (clientsocket, address) + else: + pass + msg = "Connection from {!s}({!s}). Use this information to send data to " \ + "the interface '{!s}:{:d}'.".format(address, clientsocket, host, port) + self._feedback_collect(msg, self.General_Info_ID, error=0) + except socket.timeout: pass except OSError as e: @@ -758,8 +784,9 @@ def _server_main(self, serversocket, host, port, func): # For SOCK_RAW and SOCK_DGRAM def _raw_server_main(self, serversocket, host, port, sock_type, func, sending_event, notif_host_event): - while True: + _first_client = {} + while True: sending_event.wait() sending_event.clear() if self.stop_event.is_set(): @@ -771,32 +798,39 @@ def _raw_server_main(self, serversocket, host, port, sock_type, func, notif_host_event.set() - target_address, wait_for_client = self._server_mode_additional_info[(host, port)] + target_address, wait_for_client, _ = self._server_mode_additional_info[(host, port)] if func == self._handle_connection_to_fbk_server: # args = fbk_id, fbk_length, connected_client_event assert args[0] in self._additional_fbk_desc - wait_before_sending = False + wait_before_first_sending = False elif func == self._handle_target_connection: # args = data, host, port, connected_client_event, from_fmk if args[0] is None: # In the case 'data' is None there is no data to send, # thus we are requested to only collect feedback - wait_before_sending = False + wait_before_first_sending = False elif target_address is not None: - wait_before_sending = wait_for_client + wait_before_first_sending = wait_for_client elif sock_type == socket.SOCK_RAW: # in this case target_address is not provided, but it is OK if it is a SOCK_RAW - wait_before_sending = wait_for_client + wait_before_first_sending = wait_for_client else: - wait_before_sending = True + wait_before_first_sending = True else: raise ValueError retry = 0 while retry < 10: + saved_addr = _first_client.get((host, port), None) try: - if wait_before_sending: + if saved_addr is not None: + data, address = None, saved_addr + elif wait_before_first_sending: data, address = serversocket.recvfrom(self.CHUNK_SZ) + _first_client[(host, port)] = address + msg = "Received data from {!s}. Use this information to send data to " \ + "the interface '{!s}:{:d}'.".format(address, host, port) + self._feedback_collect(msg, self.General_Info_ID, error=0) else: data, address = None, None except socket.timeout: @@ -810,11 +844,6 @@ def _raw_server_main(self, serversocket, host, port, sock_type, func, continue else: raise - except socket.error as serr: - if serr.errno == 11: # [Errno 11] Resource temporarily unavailable - retry += 1 - time.sleep(0.5) - continue else: address = address if target_address is None else target_address serversocket.settimeout(self.feedback_timeout) @@ -920,7 +949,7 @@ def _check_and_handle_obsolete_socket(skt, error=None, error_list=None): retry = 0 socket_timed_out = False - while retry < 3: + while retry < 10: try: chunk = s.recv(sz) except socket.timeout: @@ -953,7 +982,6 @@ def _check_and_handle_obsolete_socket(skt, error=None, error_list=None): has_read = True - if fbk_sockets: for s in fbk_sockets: if s in ready_to_read: @@ -976,9 +1004,10 @@ def _check_and_handle_obsolete_socket(skt, error=None, error_list=None): for s, chks in chunks.items(): fbk = b'\n'.join(chks) with self._fbk_handling_lock: - fbkid = fbk_ids[s] - fbk, err = self._feedback_handling(fbk, fbkid) - self._feedback_collect(fbk, fbkid, error=err) + if fbk != b'': + fbkid = fbk_ids[s] + fbk, err = self._feedback_handling(fbk, fbkid) + self._feedback_collect(fbk, fbkid, error=err) if (self._additional_fbk_sockets is None or s not in self._additional_fbk_sockets) and \ (self._hclient_sock2hp is None or s not in self._hclient_sock2hp.keys()) and \ (self._last_client_sock2hp is None or s not in self._last_client_sock2hp.keys()): @@ -1031,6 +1060,7 @@ def _send_data(self, sockets, data_refs, sid, from_fmk, pre_fbk=None): fbk_ids[s] = self._default_fbk_id[(host, port)] fbk_lengths[s] = self.feedback_length + assert from_fmk self._start_fbk_collector(fbk_sockets, fbk_ids, fbk_lengths, epobj, fileno2fd, from_fmk, pre_fbk=pre_fbk) @@ -1089,9 +1119,9 @@ def _send_data(self, sockets, data_refs, sid, from_fmk, pre_fbk=None): fbk_ids[s] = self._default_fbk_id[(host, port)] fbk_lengths[s] = self.feedback_length - - self._start_fbk_collector(fbk_sockets, fbk_ids, fbk_lengths, epobj, fileno2fd, from_fmk, - pre_fbk=pre_fbk) + if from_fmk: + self._start_fbk_collector(fbk_sockets, fbk_ids, fbk_lengths, epobj, fileno2fd, from_fmk, + pre_fbk=pre_fbk) else: raise TargetStuck("system not ready for sending data!") @@ -1172,10 +1202,7 @@ def is_target_ready_for_new_data(self): # We answer we are ready if at least one receiver has # terminated its job, either because the target answered to # it, or because of the current specified timeout. - if self._feedback_handled: - return True - else: - return False + return self._feedback_handled def _register_last_ack_date(self, ack_date): self._last_ack_date = ack_date diff --git a/framework/targets/printer.py b/framework/targets/printer.py old mode 100644 new mode 100755 index 2c601b0..e961ec1 --- a/framework/targets/printer.py +++ b/framework/targets/printer.py @@ -21,14 +21,16 @@ # ################################################################################ -import cups import os import random from framework.global_resources import workspace_folder -from framework.target_helpers import Target, TargetFeedback +from framework.target_helpers import Target +from framework.knowledge.feedback_collector import FeedbackCollector from libs.external_modules import cups_module +if cups_module: + import cups class PrinterTarget(Target): @@ -38,51 +40,55 @@ class PrinterTarget(Target): def __init__(self, tmpfile_ext): Target.__init__(self) - self.__suffix = '{:0>12d}'.format(random.randint(2**16, 2**32)) - self.__feedback = TargetFeedback() - self.__target_ip = None - self.__target_port = None - self.__printer_name = None - self.__cpt = None + self._suffix = '{:0>12d}'.format(random.randint(2 ** 16, 2 ** 32)) + self._feedback = FeedbackCollector() + self._target_ip = None + self._target_port = None + self._printer_name = None + self._cpt = None self.set_tmp_file_extension(tmpfile_ext) + def get_description(self): + printer_name = ', Name: ' + self._printer_name if self._printer_name is not None else '' + return 'IP: ' + self._target_ip + printer_name + def set_tmp_file_extension(self, tmpfile_ext): self._tmpfile_ext = tmpfile_ext def set_target_ip(self, target_ip): - self.__target_ip = target_ip + self._target_ip = target_ip def get_target_ip(self): - return self.__target_ip + return self._target_ip def set_target_port(self, target_port): - self.__target_port = target_port + self._target_port = target_port def get_target_port(self): - return self.__target_port + return self._target_port def set_printer_name(self, printer_name): - self.__printer_name = printer_name + self._printer_name = printer_name def get_printer_name(self): - return self.__printer_name + return self._printer_name def start(self): - self.__cpt = 0 + self._cpt = 0 if not cups_module: print('/!\\ ERROR /!\\: the PrinterTarget has been disabled because python-cups module is not installed') return False - if not self.__target_ip: + if not self._target_ip: print('/!\\ ERROR /!\\: the PrinterTarget IP has not been set') return False - if self.__target_port is None: - self.__target_port = 631 + if self._target_port is None: + self._target_port = 631 - cups.setServer(self.__target_ip) - cups.setPort(self.__target_port) + cups.setServer(self._target_ip) + cups.setPort(self._target_port) self.__connection = cups.Connection() @@ -92,16 +98,16 @@ def start(self): print('CUPS Server Errror: ', err) return False - if self.__printer_name is not None: + if self._printer_name is not None: try: - params = printers[self.__printer_name] + params = printers[self._printer_name] except: - print("Printer '%s' is not connected to CUPS server!" % self.__printer_name) + print("Printer '%s' is not connected to CUPS server!" % self._printer_name) return False else: - self.__printer_name, params = printers.popitem() + self._printer_name, params = printers.popitem() - print("\nDevice-URI: %s\nPrinter Name: %s" % (params["device-uri"], self.__printer_name)) + print("\nDevice-URI: %s\nPrinter Name: %s" % (params["device-uri"], self._printer_name)) return True @@ -109,15 +115,15 @@ def send_data(self, data, from_fmk=False): data = data.to_bytes() wkspace = workspace_folder - file_name = os.path.join(wkspace, 'fuzz_test_' + self.__suffix + self._tmpfile_ext) + file_name = os.path.join(wkspace, 'fuzz_test_' + self._suffix + self._tmpfile_ext) with open(file_name, 'wb') as f: f.write(data) - inc = '_{:0>5d}'.format(self.__cpt) - self.__cpt += 1 + inc = '_{:0>5d}'.format(self._cpt) + self._cpt += 1 try: - self.__connection.printFile(self.__printer_name, file_name, 'job_'+ self.__suffix + inc, {}) + self.__connection.printFile(self._printer_name, file_name, 'job_' + self._suffix + inc, {}) except cups.IPPError as err: print('CUPS Server Errror: ', err) \ No newline at end of file diff --git a/framework/targets/sim.py b/framework/targets/sim.py index c623926..b73cf0a 100644 --- a/framework/targets/sim.py +++ b/framework/targets/sim.py @@ -83,7 +83,7 @@ def start(self): fbk = self._retrieve_feedback_from_serial(timeout=1) code = 0 if fbk.find(b'ERROR') == -1 else -1 - self._logger.collect_target_feedback(fbk, status_code=code) + self._logger.collect_feedback(fbk, status_code=code) if code < 0: self._logger.print_console(cpin_fbk+fbk, rgb=Color.ERROR) @@ -130,4 +130,4 @@ def send_data(self, data, from_fmk=False): fbk = self._retrieve_feedback_from_serial() code = 0 if fbk.find(b'ERROR') == -1 else -1 - self._logger.collect_target_feedback(fbk, status_code=code) \ No newline at end of file + self._logger.collect_feedback(fbk, status_code=code) \ No newline at end of file diff --git a/framework/value_types.py b/framework/value_types.py index c74573f..21f80a9 100644 --- a/framework/value_types.py +++ b/framework/value_types.py @@ -39,19 +39,17 @@ if sys.version_info[0] > 2: # python3 import builtins + CHR_compat = chr else: # python2.7 import __builtin__ as builtins - -import six -from six import with_metaclass - -sys.path.append('.') + CHR_compat = unichr import framework.basic_primitives as bp from framework.encoders import * from framework.error_handling import * from framework.global_resources import * +from framework.knowledge.information import * import libs.debug_facility as dbg @@ -63,6 +61,7 @@ class VT(object): ''' mini = None maxi = None + knowledge_source = None BigEndian = 1 LittleEndian = 2 @@ -89,7 +88,23 @@ def make_random(self): pass def get_value(self): - raise NotImplementedError('New value type shall impplement this method!') + """ + Walk other the values of the object on a per-call basis. + + Returns: bytes + + """ + raise NotImplementedError('New value type shall implement this method!') + + def get_current_value(self): + """ + Provide the current value of the object. + Should not change the state of the object except if no current values. + + Returns: bytes + + """ + raise NotImplementedError('New value type shall implement this method!') def get_current_raw_val(self): return None @@ -109,15 +124,24 @@ def set_size_from_constraints(self, size=None, encoded_size=None): def pretty_print(self, max_size=None): return None + def copy_attrs_from(self, vt): + pass + + def add_specific_fuzzy_vals(self, vals): + raise NotImplementedError + + def get_specific_fuzzy_vals(self): + raise NotImplementedError + + def get_fuzzed_vt_list(self): + return None + class VT_Alt(VT): - def __init__(self, *args, **kargs): + def __init__(self): self._fuzzy_mode = False - self.init_specific(*args, **kargs) - - def init_specific(self, *args, **kargs): - raise NotImplementedError + self._specific_fuzzy_vals = None def switch_mode(self): if self._fuzzy_mode: @@ -149,127 +173,13 @@ def _enable_normal_mode(self): def _enable_fuzz_mode(self, fuzz_magnitude=1.0): raise NotImplementedError + def add_specific_fuzzy_vals(self, vals): + if self._specific_fuzzy_vals is None: + self._specific_fuzzy_vals = [] + self._specific_fuzzy_vals += convert_to_internal_repr(vals) - -class meta_8b(type): - - compatible_class = collections.OrderedDict() - fuzzy_class = collections.OrderedDict() - - def __init__(cls, name, bases, attrs): - type.__init__(cls, name, (VT,) + bases, attrs) - cls.size = 8 - cls.compat_cls = meta_8b.compatible_class - cls.fuzzy_cls = meta_8b.fuzzy_class - - # avoid adding the class of the 'six' module - if cls.__module__ != 'six': - if 'usable' in attrs: - if cls.usable == False: - return - else: - cls.usable = True - meta_8b.compatible_class[name] = cls - - if "Fuzzy" in name: - meta_8b.fuzzy_class[name] = cls - - - -class meta_16b(type): - - compatible_class = collections.OrderedDict() - fuzzy_class = collections.OrderedDict() - - def __init__(cls, name, bases, attrs): - type.__init__(cls, name, (VT,) + bases, attrs) - cls.size = 16 - cls.compat_cls = meta_16b.compatible_class - cls.fuzzy_cls = meta_16b.fuzzy_class - - # avoid adding the class of the 'six' module - if cls.__module__ != 'six': - if 'usable' in attrs: - if cls.usable == False: - return - else: - cls.usable = True - meta_16b.compatible_class[name] = cls - - if "Fuzzy" in name: - meta_16b.fuzzy_class[name] = cls - - - -class meta_32b(type): - - compatible_class = collections.OrderedDict() - fuzzy_class = collections.OrderedDict() - - def __init__(cls, name, bases, attrs): - type.__init__(cls, name, (VT,) + bases, attrs) - cls.size = 32 - cls.compat_cls = meta_32b.compatible_class - cls.fuzzy_cls = meta_32b.fuzzy_class - - # avoid adding the class of the 'six' module - if cls.__module__ != 'six': - if 'usable' in attrs: - if cls.usable == False: - return - else: - cls.usable = True - meta_32b.compatible_class[name] = cls - - if "Fuzzy" in name: - meta_32b.fuzzy_class[name] = cls - - -class meta_64b(type): - - compatible_class = collections.OrderedDict() - fuzzy_class = collections.OrderedDict() - - def __init__(cls, name, bases, attrs): - type.__init__(cls, name, (VT,) + bases, attrs) - cls.size = 64 - cls.compat_cls = meta_64b.compatible_class - cls.fuzzy_cls = meta_64b.fuzzy_class - - # avoid adding the class of the 'six' module - if cls.__module__ != 'six': - if 'usable' in attrs: - if cls.usable == False: - return - else: - cls.usable = True - meta_64b.compatible_class[name] = cls - - if "Fuzzy" in name: - meta_64b.fuzzy_class[name] = cls - - -class meta_int_str(type): - - compatible_class = collections.OrderedDict() - fuzzy_class = collections.OrderedDict() - - def __init__(cls, name, bases, attrs): - type.__init__(cls, name, (VT,) + bases, attrs) - cls.compat_cls = meta_int_str.compatible_class - cls.fuzzy_cls = meta_int_str.fuzzy_class - - # avoid adding the class of the 'six' module - if cls.__module__ != 'six': - if 'usable' in attrs: - if cls.usable == False: - return - else: - cls.usable = True - meta_int_str.compatible_class[name] = cls - - if "Fuzzy" in name: - meta_int_str.fuzzy_class[name] = cls + def get_specific_fuzzy_vals(self): + return self._specific_fuzzy_vals class String(VT_Alt): @@ -279,13 +189,18 @@ class String(VT_Alt): Attributes: encoded_string (bool): shall be set to True by any subclass that deals with encoding - specific_fuzzing_list (list): attribute to be added by subclasses that provide + subclass_fuzzing_list (list): attribute to be added by subclasses that provide specific test cases. """ DEFAULT_MAX_SZ = 10000 encoded_string = False + ctrl_char_set = ''.join([chr(i) for i in range(0, 0x20)])+'\x7f' + printable_char_set = ''.join([chr(i) for i in range(0x20, 0x7F)]) + extended_char_set = ''.join([CHR_compat(i) for i in range(0x80, 0x100)]) + non_ctrl_char = printable_char_set + extended_char_set + def encode(self, val): """ To be overloaded by a subclass that deals with encoding. @@ -381,10 +296,10 @@ def _bytes2str(self, val): ASCII = codecs.lookup('ascii').name LATIN_1 = codecs.lookup('latin-1').name - def init_specific(self, values=None, size=None, min_sz=None, - max_sz=None, determinist=True, codec='latin-1', - extra_fuzzy_list=None, absorb_regexp=None, - alphabet=None, min_encoded_sz=None, max_encoded_sz=None, encoding_arg=None): + def __init__(self, values=None, size=None, min_sz=None, + max_sz=None, determinist=True, codec='latin-1', + extra_fuzzy_list=None, absorb_regexp=None, + alphabet=None, min_encoded_sz=None, max_encoded_sz=None, encoding_arg=None): """ Initialize the String @@ -422,6 +337,8 @@ def init_specific(self, values=None, size=None, min_sz=None, parameter should support the ``__copy__`` method. """ + VT_Alt.__init__(self) + self.drawn_val = None self.values = None @@ -698,7 +615,6 @@ def _check_sizes(self, values): else: assert(sz >= self.min_sz) - def set_description(self, values=None, size=None, min_sz=None, max_sz=None, determinist=True, codec='latin-1', extra_fuzzy_list=None, @@ -727,11 +643,7 @@ def set_description(self, values=None, size=None, min_sz=None, self.regexp = absorb_regexp if extra_fuzzy_list is not None: - self.extra_fuzzy_list = self._str2bytes(extra_fuzzy_list) - elif hasattr(self, 'specific_fuzzing_list'): - self.extra_fuzzy_list = self.specific_fuzzing_list - else: - self.extra_fuzzy_list = None + self.add_specific_fuzzy_vals(extra_fuzzy_list) if values is not None: assert isinstance(values, list) @@ -885,12 +797,6 @@ def _populate_values(self, force_max_enc_sz=False, force_min_enc_sz=False): if len(self.values) == 0: raise DataModelDefinitionError - def get_current_raw_val(self, str_form=False): - if self.drawn_val is None: - self.get_value() - val = self._bytes2str(self.drawn_val) if str_form else self.drawn_val - return val - def _enable_normal_mode(self): self.values = self.values_save self.values_copy = copy.copy(self.values) @@ -901,6 +807,25 @@ def _enable_normal_mode(self): def _enable_fuzz_mode(self, fuzz_magnitude=1.0): self.values_fuzzy = [] + def add_to_fuzz_list(flist): + for v in flist: + if v not in self.values_fuzzy: + self.values_fuzzy.append(v) + + if self.knowledge_source is None \ + or not self.knowledge_source.is_info_class_represented(Language) \ + or self.knowledge_source.is_assumption_valid(Language.C): + C_strings_enabled = True + else: + C_strings_enabled = False + + if self.knowledge_source \ + and self.knowledge_source.is_info_class_represented(InputHandling) \ + and self.knowledge_source.is_assumption_valid(InputHandling.Ctrl_Char_Set): + CTRL_char_enabled = True + else: + CTRL_char_enabled = False + if self.drawn_val is not None: orig_val = self.drawn_val else: @@ -934,26 +859,24 @@ def _enable_fuzz_mode(self, fuzz_magnitude=1.0): self.values_fuzzy.append(b'\x00' * sz if sz > 0 else b'\x00') - if sz > 1: - is_even = sz % 2 == 0 - cpt = sz // 2 - if is_even: - self.values_fuzzy.append(b'%n' * cpt) - self.values_fuzzy.append(b'%s' * cpt) + if self.alphabet is not None and sz > 0: + if self.codec == self.ASCII: + base_char_set = set(self.printable_char_set) else: - self.values_fuzzy.append(orig_val[:1] + b'%n' * cpt) - self.values_fuzzy.append(orig_val[:1] + b'%s' * cpt) + base_char_set = set(self.non_ctrl_char) - self.values_fuzzy.append(orig_val + b'%n' * int(400*fuzz_magnitude)) - self.values_fuzzy.append(orig_val + b'%s' * int(400*fuzz_magnitude)) - self.values_fuzzy.append(orig_val + b'\"%n\"' * int(400*fuzz_magnitude)) - self.values_fuzzy.append(orig_val + b'\"%s\"' * int(400*fuzz_magnitude)) - self.values_fuzzy.append(orig_val + b'\r\n' * int(100*fuzz_magnitude)) + unsupported_chars = base_char_set - set(self._bytes2str(self.alphabet)) + if unsupported_chars: + sample = random.sample(unsupported_chars, 1)[0] + test_case = orig_val[:-1] + sample.encode(self.codec) + self.values_fuzzy.append(test_case) - if self.extra_fuzzy_list: - for v in self.extra_fuzzy_list: - if v not in self.values_fuzzy: - self.values_fuzzy.append(v) + self.values_fuzzy += String.fuzz_cases_ctrl_chars(self.knowledge_source, orig_val, sz, + self.max_sz, self.codec) + self.values_fuzzy += String.fuzz_cases_c_strings(self.knowledge_source, orig_val, sz, + fuzz_magnitude) + + self.values_fuzzy.append(orig_val + b'\r\n' * int(100*fuzz_magnitude)) if self.codec == self.ASCII: val = bytearray(orig_val) @@ -978,12 +901,75 @@ def _enable_fuzz_mode(self, fuzz_magnitude=1.0): if enc_cases: self.values_fuzzy += enc_cases + specif = self.get_specific_fuzzy_vals() + if specif: + add_to_fuzz_list(specif) + if hasattr(self, 'subclass_fuzzing_list'): + add_to_fuzz_list(self.subclass_fuzzing_list) + self.values_save = self.values self.values = self.values_fuzzy self.values_copy = copy.copy(self.values) self.drawn_val = None + @staticmethod + def fuzz_cases_c_strings(knowledge, orig_val, sz, fuzz_magnitude): + if knowledge is None \ + or not knowledge.is_info_class_represented(Language) \ + or knowledge.is_assumption_valid(Language.C): + + fuzzy_values = [] + + if sz > 1: + is_even = sz % 2 == 0 + cpt = sz // 2 + if is_even: + fuzzy_values.append(b'%n' * cpt) + fuzzy_values.append(b'%s' * cpt) + else: + fuzzy_values.append(orig_val[:1] + b'%n' * cpt) + fuzzy_values.append(orig_val[:1] + b'%s' * cpt) + else: + fuzzy_values.append(b'%n') + fuzzy_values.append(b'%s') + + fuzzy_values.append(orig_val + b'%n' * int(400*fuzz_magnitude)) + fuzzy_values.append(orig_val + b'%s' * int(400*fuzz_magnitude)) + fuzzy_values.append(orig_val + b'\"%n\"' * int(400*fuzz_magnitude)) + fuzzy_values.append(orig_val + b'\"%s\"' * int(400*fuzz_magnitude)) + + return fuzzy_values + + else: + return [] + + + @staticmethod + def fuzz_cases_ctrl_chars(knowledge, orig_val, sz, max_sz, codec): + if knowledge \ + and knowledge.is_info_class_represented(InputHandling) \ + and knowledge.is_assumption_valid(InputHandling.Ctrl_Char_Set): + + fuzzy_values = [] + + if sz > 0: + fuzzy_values.append(b'\x08'*max_sz) # backspace characters + if sz == max_sz: + # also include fixed size string i.e., self.min_sz == self.max_sz + for c in String.ctrl_char_set: + test_case = orig_val[:-1]+c.encode(codec) + fuzzy_values.append(test_case) + else: + for c in String.ctrl_char_set: + test_case = orig_val+c.encode(codec) + fuzzy_values.append(test_case) + + return fuzzy_values + + else: + return [] + def get_value(self): if not self.values: self._populate_values(force_max_enc_sz=self.max_enc_sz_provided, @@ -1002,6 +988,17 @@ def get_value(self): ret = self.encode(ret) return ret + def get_current_raw_val(self, str_form=False): + if self.drawn_val is None: + self.get_value() + val = self._bytes2str(self.drawn_val) if str_form else self.drawn_val + return val + + def get_current_value(self): + if self.drawn_val is None: + self.get_value() + return self.encode(self.drawn_val) if self.encoded_string else self.drawn_val + def is_exhausted(self): if self.values_copy: return False @@ -1052,6 +1049,7 @@ class INT(VT): mini = None maxi = None cformat = None + alt_cformat = None endian = None determinist = True @@ -1062,21 +1060,28 @@ class INT(VT): GEN_MIN_INT = -2**32 # 'mini_gen' is set to this when the INT subclass does not define 'mini' # and that mini is not specified by the user + fuzzy_values = None + value_space_size = None + size = None def __init__(self, values=None, min=None, max=None, default=None, determinist=True, - force_mode=False): + force_mode=False, fuzz_mode=False, values_desc=None): self.idx = 0 self.determinist = determinist self.exhausted = False self.drawn_val = None self.default = None + self._specific_fuzzy_vals = None + self.values_desc = values_desc + + if self.values_desc and not isinstance(self.values_desc, dict): + raise ValueError('@values_desc should be a dictionary') if not self.usable: raise DataModelDefinitionError("ERROR: {!r} is not usable! (use a subclass of it)" .format(self.__class__)) if values: - assert default is None if force_mode: new_values = [] for v in values: @@ -1084,13 +1089,22 @@ def __init__(self, values=None, min=None, max=None, default=None, determinist=Tr if v > self.__class__.maxi: v = self.__class__.maxi new_values.append(v) - self.values = new_values + values = new_values + elif fuzz_mode: + for v in values: + if not self.is_size_compatible(v): + raise DataModelDefinitionError("Incompatible value ({!r}) regarding possible" + " encoding size for {!s}".format(v, self.__class__)) else: for v in values: if not self.is_compatible(v): raise DataModelDefinitionError("Incompatible value ({!r}) with {!s}".format(v, self.__class__)) - self.values = list(values) + self.values = list(values) + if default is not None: + assert default in self.values + self.values.remove(default) + self.values.insert(0, default) self.values_copy = list(self.values) else: @@ -1150,6 +1164,76 @@ def make_private(self, forget_current_state): else: self.values_copy = copy.copy(self.values_copy) + def copy_attrs_from(self, vt): + self.endian = vt.endian + + def get_fuzzed_vt_list(self): + + specific_fuzzy_values = self.get_specific_fuzzy_vals() + supp_list = specific_fuzzy_values if specific_fuzzy_values is not None else [] + + val = self.get_current_raw_val() + if val is not None: + # don't use a set to preserve determinism if needed + if val-1 not in supp_list: + supp_list.append(val+1) + if val-1 not in supp_list: + supp_list.append(val-1) + + if self.values is not None: + orig_set = set(self.values) + max_oset = max(orig_set) + min_oset = min(orig_set) + if min_oset != max_oset: + diff_sorted = sorted(set(range(min_oset, max_oset+1)) - orig_set) + if diff_sorted: + item1 = diff_sorted[0] + item2 = diff_sorted[-1] + if item1 not in supp_list: + supp_list.append(item1) + if item2 not in supp_list: + supp_list.append(item2) + if max_oset+1 not in supp_list: + supp_list.append(max_oset+1) + if min_oset-1 not in supp_list: + supp_list.append(min_oset-1) + + if self.mini is not None: + cond1 = False + if self.value_space_size != -1: # meaning not an INT_str + cond1 = (self.mini != 0 or self.maxi != ((1 << self.size) - 1)) and \ + (self.mini != -(1 << (self.size-1)) or self.maxi != ((1 << (self.size-1)) - 1)) + else: + cond1 = True + + if cond1: + # we avoid using vt.mini or vt.maxi has they could be undefined (e.g., INT_str) + if self.mini_gen-1 not in supp_list: + supp_list.append(self.mini_gen-1) + if self.maxi_gen+1 not in supp_list: + supp_list.append(self.maxi_gen+1) + + if self.fuzzy_values: + for v in self.fuzzy_values: + if v not in supp_list: + supp_list.append(v) + + if supp_list: + supp_list = list(filter(self.is_size_compatible, supp_list)) + fuzzed_vt = self.__class__(values=supp_list, fuzz_mode=True) + return [fuzzed_vt] + + else: + return None + + + def add_specific_fuzzy_vals(self, vals): + if self._specific_fuzzy_vals is None: + self._specific_fuzzy_vals = [] + self._specific_fuzzy_vals += vals + + def get_specific_fuzzy_vals(self): + return self._specific_fuzzy_vals def absorb_auto_helper(self, blob, constraints): off = 0 @@ -1190,14 +1274,31 @@ def do_absorb(self, blob, constraints, off=0, size=None): raise ValueError('contents not valid!') self.values.insert(0, orig_val) self.values_copy = copy.copy(self.values) + elif self.maxi is None and self.mini is None: + # this case means 'self' is an unlimited INT (like INT_str subclass) where no constraints + # have been provided to the constructor, like INT_str(). + self.values = [orig_val] + self.values_copy = [orig_val] else: if constraints[AbsCsts.Contents]: if self.maxi is not None and orig_val > self.maxi: raise ValueError('contents not valid! (max limit)') if self.mini is not None and orig_val < self.mini: raise ValueError('contents not valid! (min limit)') - # self.values = [orig_val] - self.idx = orig_val - self.mini + else: + # mini_gen and maxi_gen are always defined + if orig_val < self.mini_gen: + if self.__class__.mini is not None and orig_val < self.__class__.mini: + raise ValueError('The type {!s} is not able to represent the value {:d}' + .format(self.__class__, orig_val)) + self.mini = self.mini_gen = orig_val + if orig_val > self.maxi_gen: + if self.__class__.maxi is not None and orig_val > self.__class__.maxi: + raise ValueError('The type {!s} is not able to represent the value {:d}' + .format(self.__class__, orig_val)) + self.maxi = self.maxi_gen = orig_val + + self.idx = orig_val - self.mini_gen # self.reset_state() self.exhausted = False @@ -1238,6 +1339,14 @@ def get_current_raw_val(self): def is_compatible(self, integer): return self.mini <= integer <= self.maxi + def is_size_compatible(self, integer): + if self.value_space_size == -1: + return True + elif -((self.value_space_size + 1) // 2) <= integer <= self.value_space_size: + return True + else: + return False + def set_value_list(self, new_list): ret = False if self.values: @@ -1332,7 +1441,7 @@ def get_value(self): self.drawn_val = val return self._convert_value(val) - def get_current_encoded_value(self): + def get_current_value(self): if self.drawn_val is None: self.get_value() return self._convert_value(self.drawn_val) @@ -1344,12 +1453,18 @@ def pretty_print(self, max_size=None): if self.drawn_val is None: self.get_value() + if self.values_desc: + desc = self.values_desc.get(self.drawn_val) + desc = '' if desc is None else ' [' + desc + ']' + else: + desc = '' + if self.drawn_val < 0: formatted_val = '-0x' + hex(self.drawn_val)[3:].upper() else: formatted_val = '0x' + hex(self.drawn_val)[2:].upper() - return str(self.drawn_val) + ' (' + formatted_val + ')' + return str(self.drawn_val) + ' (' + formatted_val + ')' + desc def rewind(self): @@ -1369,7 +1484,13 @@ def _unconvert_value(self, val): return struct.unpack(self.cformat, val)[0] def _convert_value(self, val): - return struct.pack(self.cformat, val) + try: + string = struct.pack(self.cformat, val) + except: + # this case can trigger with values that have been added for fuzzing + string = struct.pack(self.alt_cformat, val) + + return string def _read_value_from(self, blob, size): sz = struct.calcsize(self.cformat) @@ -1418,13 +1539,32 @@ def is_exhausted(self): class Filename(String): - specific_fuzzing_list = [ - b'../../../../../../etc/password', - b'../../../../../../Windows/system.ini', - b'file%n%n%n%nname.txt', - ] + @property + def subclass_fuzzing_list(self): + linux_spe = [b'../../../../../../etc/password'] + windows_spe = [b'..\\..\\..\\..\\..\\..\\Windows\\system.ini'] + c_spe = [b'file%n%n%n%nname.txt'] + if self.knowledge_source is None: + flist = linux_spe+windows_spe+c_spe + else: + flist = [] + if self.knowledge_source.is_info_class_represented(OS): + if self.knowledge_source.is_assumption_valid(OS.Linux): + flist += linux_spe + if self.knowledge_source.is_assumption_valid(OS.Windows): + flist += windows_spe + else: + flist = linux_spe+windows_spe + if self.knowledge_source.is_info_class_represented(Language): + if self.knowledge_source.is_assumption_valid(Language.C): + flist += c_spe + else: + flist += c_spe + + return flist + def from_encoder(encoder_cls, encoding_arg=None): def internal_func(string_subclass): def new_meth(meth): @@ -1451,68 +1591,138 @@ class GSMPhoneNum(String): pass class Wrapper(String): pass -class Fuzzy_INT(INT): - ''' - Base class to be inherited and not used directly - ''' - values = None - short_cformat = None +class INT_str(INT): + endian = VT.Native + usable = True - def __init__(self, endian=VT.BigEndian, supp_list=None): - self.endian = endian - if supp_list: - self.extend_value_list(supp_list) + regex_decimal = b'-?\d+' - assert(self.values is not None) - INT.__init__(self, values=self.values, determinist=True) + regex_upper_hex = b'-?[0123456789ABCDEF]+' + regex_lower_hex = b'-?[0123456789abcdef]+' - def make_private(self, forget_current_state): - self.values = copy.copy(self.values) + regex_octal = b'-?[01234567]+' - def is_compatible(self, integer): - if self.mini <= integer <= self.maxi: - return True - elif -((self.maxi + 1) // 2) <= integer <= ((self.maxi + 1) // 2) - 1: - return True + regex_bin = b'-?[01]+' + + fuzzy_values = [0, -1, -2**32, 2 ** 32 - 1, 2 ** 32] + value_space_size = -1 # means infinite + + def __init__(self, values=None, min=None, max=None, default=None, determinist=True, + force_mode=False, fuzz_mode=False, base=10, letter_case='upper', min_size=None, reverse=False): + INT.__init__(self, values=values, min=min, max=max, default=default, determinist=determinist, + force_mode=force_mode, fuzz_mode=fuzz_mode) + assert base in [10, 16, 8, 2] + assert letter_case in ['upper', 'lower'] + assert min_size is None or isinstance(min_size, int) + + self._base = base + self._reverse = reverse + self._min_size = min_size + self._letter_case = letter_case + + self._format_str, self._regex = self._prepare_format_str(self._min_size, self._base, + self._letter_case) + + def copy_attrs_from(self, vt): + INT.copy_attrs_from(self, vt) + self._base = vt._base + self._letter_case = vt._letter_case + self._min_size = vt._min_size + self._reverse = vt._reverse + self._format_str = vt._format_str + self._regex = vt._regex + + def _prepare_format_str(self, min_size, base, letter_case): + if min_size is not None: + format_str = '{:0' + str(min_size) else: - return False + format_str = '{:' + + if base == 10: + format_str += '}' + regex = self.regex_decimal + elif base == 16: + if letter_case == 'upper': + format_str += 'X}' + regex = self.regex_upper_hex + else: + format_str += 'x}' + regex = self.regex_lower_hex + elif base == 8: + format_str += 'o}' + regex = self.regex_octal + elif base == 2: + format_str += 'b}' + regex = self.regex_bin + else: + raise ValueError(self._base) - def _convert_value(self, val): - try: - string = struct.pack(VT.enc2struct[self.endian] + self.short_cformat, val) - except: - string = struct.pack(VT.enc2struct[self.endian] + self.alt_short_cformat, val) + return (format_str, regex) - return string + def get_fuzzed_vt_list(self): + vt_list = INT.get_fuzzed_vt_list(self) + fuzzed_vals = [] + def handle_size(self, v): + sz = 1 if v == 0 else math.ceil(math.log(abs(v), self._base)) + if sz <= new_min_size: + format_str, _ = self._prepare_format_str(new_min_size, + self._base, self._letter_case) + fuzzed_vals.append(format_str.format(v)) + return True + else: + return False -#class INT_str(VT, metaclass=meta_int_str): -class INT_str(with_metaclass(meta_int_str, INT)): - endian = VT.Native + if self._min_size is not None and self._min_size > 1: + new_min_size = self._min_size - 1 - def is_compatible(self, integer): - return True + if self.values: + for v in self.values: + if handle_size(self, v): + break + elif self.mini is None: + handle_size(self, 5) + elif self.mini >= 0: + ok = handle_size(self, self.mini) + if not ok: + for v in range(0, self.mini): + if handle_size(self, v): + break - def _read_value_from(self, blob, size): - g = re.match(b'-?\d+', blob) - if g is None: - raise ValueError + qty = self._min_size if self._min_size is not None else 1 + if self._base <= 10: + fuzzed_vals.append('A'*qty) else: - return g.group(0), len(g.group(0)) + if self._letter_case == 'upper': + fuzzed_vals.append('a'*qty) + else: + fuzzed_vals.append('A'*qty) + if self._base <= 8: + fuzzed_vals.append('8'*qty) + if self._base <= 2: + fuzzed_vals.append('2'*qty) - def _unconvert_value(self, val): - return int(val) + orig_val = self.get_current_value() + sz = len(orig_val) + if self._min_size and self._min_size > sz: + max_sz = self._min_size + else: + max_sz = sz - def _convert_value(self, val): - return str(val).encode('utf8') + fuzzed_vals += String.fuzz_cases_ctrl_chars(self.knowledge_source, orig_val, sz, + max_sz, codec=String.ASCII) + fuzzed_vals += String.fuzz_cases_c_strings(self.knowledge_source, orig_val, sz, + fuzz_magnitude=0.3) + + fuzzed_vals.append(orig_val + b'\r\n' * 100) + if fuzzed_vals: + if vt_list is None: + vt_list = [] + vt_list.insert(0, String(values=fuzzed_vals)) -#class Fuzzy_INT_str(Fuzzy_INT, metaclass=meta_int_str): -class Fuzzy_INT_str(with_metaclass(meta_int_str, Fuzzy_INT)): - values = [0, -1, -2**32, 2 ** 32 - 1, 2 ** 32, - b'%n'*8, b'%n'*100, b'\"%n\"'*100, - b'%s'*8, b'%s'*100, b'\"%s\"'*100] + return vt_list def is_compatible(self, integer): return True @@ -1523,15 +1733,29 @@ def pretty_print(self, max_size=None): return str(self.drawn_val) + def _read_value_from(self, blob, size): + g = re.match(self._regex, blob) + if g is None: + raise ValueError + else: + return g.group(0), len(g.group(0)) + + def _unconvert_value(self, val): + if self._reverse: + val = val[::-1] + return int(val, base=self._base) + def _convert_value(self, val): if isinstance(val, int): - return str(val).encode('utf8') + ret = self._format_str.format(val).encode('utf8') + if self._reverse: + ret = ret[::-1] + return ret else: + # for some fuzzing case assert isinstance(val, bytes) return val - - class BitField(VT_Alt): ''' Provide: @@ -1541,11 +1765,13 @@ class BitField(VT_Alt): ''' padding_one = [0, 1, 0b11, 0b111, 0b1111, 0b11111, 0b111111, 0b1111111] - def init_specific(self, subfield_limits=None, subfield_sizes=None, - subfield_values=None, subfield_val_extremums=None, - padding=0, lsb_padding=True, - endian=VT.LittleEndian, determinist=True, - subfield_descs=None, defaults=None): + def __init__(self, subfield_limits=None, subfield_sizes=None, + subfield_values=None, subfield_val_extremums=None, + padding=0, lsb_padding=True, + endian=VT.BigEndian, determinist=True, + subfield_descs=None, defaults=None): + + VT_Alt.__init__(self) self.drawn_val = None self.exhausted = False @@ -1571,24 +1797,24 @@ def init_specific(self, subfield_limits=None, subfield_sizes=None, self.current_idx = None self.idx = None self.idx_inuse = None - self.set_bitfield(sf_valuess=subfield_values, sf_val_extremums=subfield_val_extremums, - sf_limits=subfield_limits, sf_sizes=subfield_sizes, sf_descs=subfield_descs, - sf_defaults=defaults) + self.set_bitfield(sf_values=subfield_values, sf_val_extremums=subfield_val_extremums, + sf_limits=subfield_limits, sf_sizes=subfield_sizes, + sf_descs=subfield_descs, sf_defaults=defaults) def make_private(self, forget_current_state): # no need to copy self.default (that should not be modified) - self.subfield_limits = copy.copy(self.subfield_limits) - self.subfield_sizes = copy.copy(self.subfield_sizes) - self.subfield_vals = copy.copy(self.subfield_vals) - self.subfield_vals_save = copy.copy(self.subfield_vals_save) - self.subfield_extrems = copy.copy(self.subfield_extrems) - self.subfield_extrems_save = copy.copy(self.subfield_extrems_save) + self.subfield_limits = copy.deepcopy(self.subfield_limits) + self.subfield_sizes = copy.deepcopy(self.subfield_sizes) + self.subfield_vals = copy.deepcopy(self.subfield_vals) + self.subfield_vals_save = copy.deepcopy(self.subfield_vals_save) + self.subfield_extrems = copy.deepcopy(self.subfield_extrems) + self.subfield_extrems_save = copy.deepcopy(self.subfield_extrems_save) if forget_current_state: self.reset_state() else: - self.idx = copy.copy(self.idx) - self.idx_inuse = copy.copy(self.idx_inuse) - self.subfield_fuzzy_vals = copy.copy(self.subfield_fuzzy_vals) + self.idx = copy.deepcopy(self.idx) + self.idx_inuse = copy.deepcopy(self.idx_inuse) + self.subfield_fuzzy_vals = copy.deepcopy(self.subfield_fuzzy_vals) def reset_state(self): self._reset_idx() @@ -1633,10 +1859,10 @@ def set_subfield(self, idx, val): self.subfield_extrems[idx][1] = builtins.max(maxi, val) self.idx_inuse[idx] = self.idx[idx] = val - mini else: - # Note that the case "self.idx[idx]==1" has not to be - # specifically handled here (for preventing overflow), - # because even if len(subfield_vals)==1, we add a new element - # within, making a subfield_vals always >= 2. + try: + self.subfield_vals[idx].remove(val) + except: + pass self.subfield_vals[idx].insert(self.idx[idx], val) self.idx_inuse[idx] = self.idx[idx] @@ -1657,7 +1883,7 @@ def get_subfield(self, idx): return ret - def set_bitfield(self, sf_valuess=None, sf_val_extremums=None, sf_limits=None, sf_sizes=None, + def set_bitfield(self, sf_values=None, sf_val_extremums=None, sf_limits=None, sf_sizes=None, sf_descs=None, sf_defaults=None): if sf_limits is not None: @@ -1670,9 +1896,9 @@ def set_bitfield(self, sf_valuess=None, sf_val_extremums=None, sf_limits=None, s else: raise DataModelDefinitionError - if sf_valuess is None: - sf_valuess = [None for i in range(len(self.subfield_limits))] - elif len(sf_valuess) != len(self.subfield_limits): + if sf_values is None: + sf_values = [None for i in range(len(self.subfield_limits))] + elif len(sf_values) != len(self.subfield_limits): raise DataModelDefinitionError if sf_val_extremums is None: @@ -1707,19 +1933,22 @@ def set_bitfield(self, sf_valuess=None, sf_val_extremums=None, sf_limits=None, s # provided limits are not included in the subfields for idx, lim in enumerate(self.subfield_limits): - values = sf_valuess[idx] + values = sf_values[idx] extrems = sf_val_extremums[idx] size = lim - prev_lim self.subfield_sizes.append(size) if values is not None: - default = self.subfield_defaults[idx] - assert default is None l = [] for v in values: if self.is_compatible(v, size): l.append(v) + default = self.subfield_defaults[idx] + if default is not None: + assert default in l + l.remove(default) + l.insert(self.idx[idx], default) self.subfield_vals.append(l) self.subfield_extrems.append(None) else: @@ -1744,7 +1973,16 @@ def set_bitfield(self, sf_valuess=None, sf_val_extremums=None, sf_limits=None, s self.subfield_fuzzy_vals.append(None) prev_lim = lim - def extend_right(self, bitfield): + + @property + def bit_length(self): + return self.size + + @property + def byte_length(self): + return self.nb_bytes + + def extend(self, bitfield, rightside=True): if self.drawn_val is None: self.get_current_value() @@ -1777,17 +2015,23 @@ def extend_right(self, bitfield): else: term2 = bitfield.drawn_val - self.drawn_val = (term2 << self.size) + term1 + if rightside: + self.drawn_val = (term2 << self.size) + term1 + else: + self.drawn_val = (term1 << bitfield.size) + term2 sz_mod = (self.size + bitfield.size) % 8 new_padding_sz = 8 - sz_mod if sz_mod != 0 else 0 if self.lsb_padding: self.drawn_val <<= new_padding_sz - self.current_val_update_pending = False - self.idx += bitfield.idx - self.idx_inuse += bitfield.idx_inuse + if rightside: + self.idx += bitfield.idx + self.idx_inuse += bitfield.idx_inuse + else: + self.idx = bitfield.idx + self.idx + self.idx_inuse = bitfield.idx_inuse + self.idx_inuse if self.subfield_descs is not None or bitfield.subfield_descs is not None: if self.subfield_descs is None and bitfield.subfield_descs is not None: @@ -1795,18 +2039,35 @@ def extend_right(self, bitfield): desc_extension = bitfield.subfield_descs elif self.subfield_descs is not None and bitfield.subfield_descs is None: desc_extension = [None for i in bitfield.subfield_limits] - self.subfield_descs += desc_extension - - self.subfield_sizes += bitfield.subfield_sizes - self.subfield_vals += bitfield.subfield_vals - self.subfield_extrems += bitfield.subfield_extrems - self.subfield_defaults += bitfield.subfield_defaults + if rightside: + self.subfield_descs += desc_extension + else: + self.subfield_descs = desc_extension + self.subfield_descs - for l in bitfield.subfield_limits: - self.subfield_limits.append(self.size + l) + if rightside: + self.subfield_sizes += bitfield.subfield_sizes + self.subfield_vals += bitfield.subfield_vals + self.subfield_extrems += bitfield.subfield_extrems + self.subfield_defaults += bitfield.subfield_defaults + + for l in bitfield.subfield_limits: + self.subfield_limits.append(self.size + l) + + self.subfield_fuzzy_vals += bitfield.subfield_fuzzy_vals + + else: + self.subfield_sizes = bitfield.subfield_sizes + self.subfield_sizes + self.subfield_vals = bitfield.subfield_vals + self.subfield_vals + self.subfield_extrems = bitfield.subfield_extrems + self.subfield_extrems + self.subfield_defaults = bitfield.subfield_defaults + self.subfield_defaults + + supp_limits = [] + for l in self.subfield_limits: + supp_limits.append(bitfield.size + l) + self.subfield_limits = bitfield.subfield_limits + supp_limits + + self.subfield_fuzzy_vals = bitfield.subfield_fuzzy_vals + self.subfield_fuzzy_vals - self.subfield_fuzzy_vals += bitfield.subfield_fuzzy_vals - self.size = self.subfield_limits[-1] self.nb_bytes = int(math.ceil(self.size / 8.0)) @@ -1815,6 +2076,11 @@ def extend_right(self, bitfield): else: self.padding_size = 8 - (self.size % 8) + def extend_right(self, bitfield): + self.extend(bitfield, rightside=True) + + def extend_left(self, bitfield): + self.extend(bitfield, rightside=False) def set_size_from_constraints(self, size=None, encoded_size=None): raise DataModelDefinitionError @@ -1822,6 +2088,8 @@ def set_size_from_constraints(self, size=None, encoded_size=None): def pretty_print(self, max_size=None): + current_raw_val = self.get_current_raw_val() + first_pass = True for lim, sz, values, extrems, i in zip(self.subfield_limits[::-1], self.subfield_sizes[::-1], @@ -1849,7 +2117,10 @@ def pretty_print(self, max_size=None): if self.padding_size != 0: if self.padding == 1: - pad = '1'*self.padding_size + # in the case the padding has been modified, following an absorption, + # to something not standard because of AbsCsts.Contents == False, + # we use the altered padding which has been stored in self.padding_one + pad = bin(self.padding_one[self.padding_size])[2:] else: pad = '0'*self.padding_size if self.lsb_padding: @@ -1860,7 +2131,7 @@ def pretty_print(self, max_size=None): else: string += ' |-)' - return string + ' ' + str(self.get_current_raw_val()) + return string + ' ' + str(current_raw_val) def is_compatible(self, integer, size): @@ -2026,6 +2297,20 @@ def rewind(self): def _read_value_from(self, blob, size, endian, constraints): + """ + Used by .do_absorb(). + side effect: may change self.padding_one dictionary. + """ + def recompute_padding(masked_val, mask): + if masked_val != mask and masked_val != 0: + self.padding = 1 + self.padding_one = copy.copy(self.padding_one) + self.padding_one[self.padding_size] = masked_val + elif masked_val == mask and self.padding == 0: + self.padding = 1 + elif masked_val == 0 and self.padding == 1: + self.padding = 0 + values = list(struct.unpack('B'*size, blob)) if endian == VT.BigEndian: @@ -2035,19 +2320,25 @@ def _read_value_from(self, blob, size, endian, constraints): if self.padding_size != 0: if self.lsb_padding: + mask = self.padding_one[self.padding_size] if constraints[AbsCsts.Contents]: - mask = self.padding_one[self.padding_size] if self.padding == 1 and values[0] & mask != mask: raise ValueError('contents not valid! (padding should be 1s)') - elif self.padding == 0 and values[0] & self.padding_one[self.padding_size] != 0: + elif self.padding == 0 and values[0] & mask != 0: raise ValueError('contents not valid! (padding should be 0s)') + else: + masked_val = values[0] & mask + recompute_padding(masked_val, mask) else: + mask = self.padding_one[self.padding_size]<<(8-self.padding_size) if constraints[AbsCsts.Contents]: - mask = self.padding_one[self.padding_size]<<(8-self.padding_size) if self.padding == 1 and values[-1] & mask != mask: raise ValueError('contents not valid! (padding should be 1s)') elif self.padding == 0 and values[-1] & mask != 0: raise ValueError('contents not valid! (padding should be 0s)') + else: + masked_val = values[-1] & mask + recompute_padding(masked_val, mask) values_sz = len(values) result = 0 @@ -2079,6 +2370,8 @@ def do_absorb(self, blob, constraints, off=0, size=None): self.orig_idx = copy.deepcopy(self.idx) self.orig_subfield_vals = copy.deepcopy(self.subfield_vals) self.orig_drawn_val = self.drawn_val + self.orig_padding = self.padding + self.padding_one = self.__class__.padding_one self.reset_state() @@ -2123,6 +2416,8 @@ def do_revert_absorb(self): self.idx = self.orig_idx self.subfield_vals = self.orig_subfield_vals self.drawn_val = self.orig_drawn_val + self.padding = self.orig_padding + self.padding_one = self.__class__.padding_one def do_cleanup_absorb(self): ''' @@ -2132,6 +2427,7 @@ def do_cleanup_absorb(self): del self.orig_idx del self.orig_subfield_vals del self.orig_drawn_val + del self.orig_padding def get_value(self): ''' @@ -2315,392 +2611,139 @@ def is_exhausted(self): return self.exhausted -#class INT8(INT, metaclass=meta_8b): -class INT8(with_metaclass(meta_8b, INT)): + +class INT8(INT): + fuzzy_values = [0xFF, 0, 0x01, 0x80, 0x7F] + value_space_size = 2**8-1 + size = 8 usable = False class SINT8(INT8): mini = -2**7 maxi = 2**7-1 cformat = 'b' + alt_cformat = 'B' endian = VT.Native + usable = True class UINT8(INT8): mini = 0 maxi = 2**8-1 cformat = 'B' + alt_cformat = 'b' endian = VT.Native + usable = True -#class Fuzzy_INT8(Fuzzy_INT, metaclass=meta_8b): -class Fuzzy_INT8(with_metaclass(meta_8b, Fuzzy_INT)): - mini = 0 - maxi = 2**8-1 - values = [0xFF, 0, 0x01, 0x80, 0x7F] - short_cformat = 'B' - alt_short_cformat = 'b' - - -#class INT16(VT, metaclass=meta_16b): -class INT16(with_metaclass(meta_16b, INT)): +class INT16(INT): + fuzzy_values = [0xFFFF, 0, 0x8000, 0x7FFF] + value_space_size = 2**16-1 + size = 16 usable = False - class SINT16_be(INT16): mini = -2**15 maxi = 2**15-1 cformat = '>h' + alt_cformat = '>H' endian = VT.BigEndian + usable = True class SINT16_le(INT16): mini = -2**15 maxi = 2**15-1 cformat = ' New String\n') - - t = String(values=['AAA', 'BBBB', 'CCCCC'], min_sz=3, max_sz=10) - - for i in range(30): - print(t.get_value()) - if t.is_exhausted(): - break - - print('\n********\n') - t.reset_state() - t.switch_mode() - - for i in range(30): - print(t.get_value()) - if t.is_exhausted(): - break - - print('\n********\n') - t.reset_state() - t.switch_mode() - - for i in range(30): - print(t.get_value()) - if t.is_exhausted(): - break - - print('\n********\n') - - t.reset_state() - t.get_value() - t.get_value() - t.switch_mode() - - for i in range(30): - print(t.get_value()) - if t.is_exhausted(): - break - - print('\n====> New String\n') - - t = String(values=['AAA', 'BBBB', 'CCCCC'], max_sz=10) - - print(t.get_value()) - print(t.get_value()) - - print('\n********\n') - - t.rewind() - print(t.get_value()) - print(t.get_value()) - - print('\n********\n') - - t.reset_state() - print(t.get_value()) - print(t.get_value()) - - print('\n********\n') - - t.rewind() - t.rewind() - print(t.get_value()) - print(t.get_value()) - - print('\n********\n') - - t.rewind() - t.rewind() - t.rewind() - t.rewind() - print(t.get_value()) - print(t.get_value()) - - print('\n====> New String\n') - - t = String(min_sz=1, max_sz=10) - - for i in range(30): - print(t.get_value()) - if t.is_exhausted(): - break - - print('\n********\n') - t.reset_state() - t.switch_mode() - - for i in range(30): - print(t.get_value()) - if t.is_exhausted(): - break - - print('\n********\n') - - t.rewind() - t.rewind() - print(t.get_value()) - print(t.get_value()) + usable = True \ No newline at end of file diff --git a/libs/debug_facility.py b/libs/debug_facility.py index 8061cb1..f8b7bf8 100644 --- a/libs/debug_facility.py +++ b/libs/debug_facility.py @@ -34,6 +34,9 @@ # related to fuzzing_primitives.py MW_DEBUG = False +# related to knowledge infrastructure +KNOW_DEBUG = True + try: from xtermcolor import colorize except ImportError: @@ -56,3 +59,6 @@ def DEBUG_PRINT(msg, level=1, rgb=None): print(colorize(msg, rgb=DebugColor.LEVEL[level])) else: print(colorize(msg, rgb=rgb)) + +def NO_PRINT(msg, level=1, rgb=None): + return \ No newline at end of file diff --git a/libs/external_modules.py b/libs/external_modules.py index 3cd8177..13e3b6c 100644 --- a/libs/external_modules.py +++ b/libs/external_modules.py @@ -21,10 +21,19 @@ # ################################################################################ +import sys + try: import xtermcolor from xtermcolor import colorize xtermcolor.isatty = lambda x: True + + if sys.version_info[0] <= 2: + def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): + if isinstance(string, unicode): + string = str(string) + return xtermcolor.colorize(string, rgb=rgb, ansi=ansi, bg=bg, ansi_bg=ansi_bg, fd=fd) + except ImportError: print("WARNING [FMK]: python-xtermcolor module is not installed, colors won't be available!") def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): @@ -77,6 +86,11 @@ class Color(object): ND_ENCODED = 0xFFA500 ND_CUSTO = 0x800080 + ANALYSIS_CONFIRM = 0xEF0000 + ANALYSIS_FALSEPOSITIVE = 0x00FF00 + ANALYSIS_IMPACT = 0xFF0000 + ANALYSIS_NO_IMPACT = 0x00C0FF + @staticmethod def display(): for c in dir(Color): diff --git a/libs/utils.py b/libs/utils.py index 15e487b..c5de18f 100644 --- a/libs/utils.py +++ b/libs/utils.py @@ -22,8 +22,10 @@ ################################################################################ import os +import sys import subprocess import re +import inspect def ensure_dir(f): d = os.path.dirname(f) @@ -73,3 +75,13 @@ def retrieve_app_handler(filename): result = re.search("Exec=(.*)", buff) app_name = result.group(1).split()[0] return app_name + + +if sys.version_info[0] > 2: + def get_caller_object(stack_frame=2): + caller_frame_record = inspect.stack()[stack_frame] + return caller_frame_record.frame.f_locals['self'] +else: + def get_caller_object(stack_frame=2): + caller_frame_record = inspect.stack()[stack_frame] + return caller_frame_record[0].f_locals['self'] diff --git a/projects/generic/standard_proj.py b/projects/generic/standard_proj.py index 2e8b944..99b4f61 100644 --- a/projects/generic/standard_proj.py +++ b/projects/generic/standard_proj.py @@ -33,7 +33,7 @@ # If you only want one default DM, provide its name directly as follows: # project.default_dm = 'mydf' -logger = Logger('standard', export_data=False, explicit_data_recording=True, export_orig=False, +logger = Logger('standard', record_data=False, explicit_data_recording=True, export_orig=False, enable_file_logging=False) printer1_tg = PrinterTarget(tmpfile_ext='.png') @@ -47,15 +47,10 @@ class display_mem_check(ProbeMem): process_name = 'display' tolerance = 10 -local_tg = LocalTarget(tmpfile_ext='.png') -local_tg.set_target_path('/usr/bin/display') - -local2_tg = LocalTarget(tmpfile_ext='.pdf') -local2_tg.set_target_path('okular') - -local3_tg = LocalTarget(tmpfile_ext='.zip') -local3_tg.set_target_path('unzip') -local3_tg.set_post_args('-d ' + gr.workspace_folder) +local_tg = LocalTarget(tmpfile_ext='.png', target_path='/usr/bin/display') +local2_tg = LocalTarget(tmpfile_ext='.pdf', target_path='okular') +local3_tg = LocalTarget(tmpfile_ext='.zip', target_path='unzip', post_args='-d ' + gr.workspace_folder) +local4_tg = LocalTarget(target_path='python', pre_args='-c', send_via_cmdline=True) net_tg = NetworkTarget(host='localhost', port=12345, data_semantics='TG1', hold_connection=True) net_tg.register_new_interface('localhost', 54321, (socket.AF_INET, socket.SOCK_STREAM), 'TG2', server_mode=True) @@ -73,13 +68,14 @@ class display_mem_check(ProbeMem): targets = [(local_tg, (display_mem_check, 0.1)), local2_tg, local3_tg, + local4_tg, printer1_tg, net_tg, netsrv_tg, rawnetsrv_tg] @operator(project, - gen_args={'init': ('make the model walker ignore all the steps until the provided one', 1, int), - 'max_steps': ("number of test cases to run", 20, int)}, - args={'mode': ('strategy mode (0 or 1)', 0, int), + args={'init': ('make the model walker ignore all the steps until the provided one', 1, int), + 'max_steps': ("number of test cases to run", 20, int), + 'mode': ('strategy mode (0 or 1)', 0, int), 'path': ("path of the target application (for LocalTarget's only)", '/usr/bin/display', str)}) class Op1(Operator): diff --git a/projects/specific/usb_proj.py b/projects/specific/usb_proj.py index edf4ed8..fabc851 100644 --- a/projects/specific/usb_proj.py +++ b/projects/specific/usb_proj.py @@ -31,7 +31,7 @@ project = Project() project.default_dm = 'usb' -logger = Logger('bin', export_data=False, explicit_data_recording=True, export_orig=False) +logger = Logger('bin', record_data=False, explicit_data_recording=True, export_orig=False) rpyc_module = True try: @@ -116,9 +116,9 @@ class PandaboardArgs(object): @operator(project, - gen_args={'init': ('make the model walker ignore all the steps until the provided one', 1, int), - 'max_steps': ("number of test cases to run", 20, int)}, - args={'mode': ('strategy mode: 0, 1 (fuzz DEV), 2 (Mass-Storage) or 666 (BigConf)', 2, int)}) + args={'init': ('make the model walker ignore all the steps until the provided one', 1, int), + 'max_steps': ("number of test cases to run", 20, int), + 'mode': ('strategy mode: 0, 1 (fuzz DEV), 2 (Mass-Storage) or 666 (BigConf)', 2, int)}) class Op1(Operator): def start(self, fmk_ops, dm, monitor, target, logger, user_input): @@ -135,15 +135,15 @@ def start(self, fmk_ops, dm, monitor, target, logger, user_input): if self.mode == 1: self.instr_list.append([('DEV', UI(finite=True)), ('tTYPE', UI(init=self.init))]) elif self.mode == 2: - self.instr_list.append([('DEV', UI(finite=True)), ('ALT', None, UI(conf='MS'))]) + self.instr_list.append([('DEV', UI(finite=True)), ('ALT', UI(conf='MS'))]) else: self.instr_list.append([('DEV', UI(finite=True))]) if self.mode == 666: - self.instr_list.append([('CONF', UI(finite=True)), ('ALT', None, UI(conf='BIGCONF')), - ('tTYPE#2', UI(init=self.init, clone_node=False), None)]) + self.instr_list.append([('CONF', UI(finite=True)), ('ALT', UI(conf='BIGCONF')), + ('tTYPE#2', UI(init=self.init, clone_node=False))]) elif self.mode == 2: - self.instr_list.append([('CONF', UI(finite=True)), ('ALT', None, UI(conf='MSD')), + self.instr_list.append([('CONF', UI(finite=True)), ('ALT', UI(conf='MSD')), ('tTYPE#2', UI(init=self.init))]) else: self.instr_list.append([('CONF', UI(finite=True)), ('tTYPE#2', UI(init=self.init))]) diff --git a/projects/tuto_proj.py b/projects/tuto_proj.py index 03d7154..f049a93 100644 --- a/projects/tuto_proj.py +++ b/projects/tuto_proj.py @@ -26,19 +26,34 @@ from framework.plumbing import * from framework.targets.debug import TestTarget from framework.targets.network import NetworkTarget +from framework.knowledge.information import * +from framework.knowledge.feedback_handler import TestFbkHandler project = Project() -project.default_dm = 'mydf' +project.default_dm = ['mydf', 'myproto'] -logger = Logger(export_data=False, explicit_data_recording=False, export_orig=False, - export_raw_data=True, enable_file_logging=False) +project.map_targets_to_scenario('ex1', {0: 7, 1: 8, None: 8}) + +logger = Logger(record_data=False, explicit_data_recording=False, export_orig=False, + export_raw_data=False, enable_file_logging=False) + +### KNOWLEDGE ### + +project.add_knowledge( + Hardware.X86_64, + Language.C +) + +project.register_feedback_handler(TestFbkHandler()) + +### TARGETS DEFINITION ### class TutoNetTarget(NetworkTarget): def _custom_data_handling_before_emission(self, data_list): self.listen_to('localhost', 64001, 'Dynamic server interface') # self.connect_to('localhost', 64002, 'Dynamic client interface') - # self._logger.collect_target_feedback('TEST', status_code=random.randint(-2,2)) + # self._logger.collect_feedback('TEST', status_code=random.randint(-2,2)) def _feedback_handling(self, fbk, ref): # self.remove_all_dynamic_interfaces() @@ -54,7 +69,7 @@ def _feedback_handling(self, fbk, ref): net_tg = NetworkTarget(host='localhost', port=12345, socket_type=(socket.AF_INET, socket.SOCK_STREAM), - hold_connection=True, server_mode=False) + hold_connection=True, server_mode=False, keep_first_client=False) udpnet_tg = NetworkTarget(host='localhost', port=12345, socket_type=(socket.AF_INET, socket.SOCK_DGRAM), @@ -83,7 +98,7 @@ def start(self, dm, target, logger): def main(self, dm, target, logger): self.cpt += 1 - return ProbeStatus(self.cpt) + return ProbeStatus(self.cpt, info='This is a Linux OS!') @probe(project) @@ -117,25 +132,29 @@ def main(self, dm, target, logger): return ProbeStatus(status) -serial_backend = Serial_Backend('/dev/ttyUSB0', username='test', password='test', slowness_factor=8) +if serial_module: + serial_backend = Serial_Backend('/dev/ttyUSB0', username='test', password='test', slowness_factor=8) -@blocking_probe(project) -class probe_pid(ProbePID): - backend = serial_backend - process_name = 'bash' - -@probe(project) -class probe_mem(ProbeMem): - backend = serial_backend - process_name = 'bash' - tolerance = 1 + @blocking_probe(project) + class probe_pid(ProbePID): + backend = serial_backend + process_name = 'bash' + @probe(project) + class probe_mem(ProbeMem): + backend = serial_backend + process_name = 'bash' + tolerance = 1 ### TARGETS ALLOCATION ### targets = [(EmptyTarget(), (P1, 2), (P2, 1.4), health_check), tuto_tg, net_tg, udpnet_tg, udpnetsrv_tg, rawnetsrv_tg, - (TestTarget(), probe_pid, (probe_mem, 0.2))] + TestTarget(fbk_samples=['CRC error', 'OK']), + TestTarget()] + +if serial_module: + targets.append((TestTarget(), probe_pid, (probe_mem, 0.2))) ### OPERATOR DEFINITION ### @@ -169,8 +188,8 @@ def plan_next_operation(self, fmk_ops, dm, monitor, target, logger, fmk_feedback op = Operation() - p1_ret = monitor.get_probe_status(P1).get_status() - p2_ret = monitor.get_probe_status(P2).get_status() + p1_ret = monitor.get_probe_status(P1).value + p2_ret = monitor.get_probe_status(P2).value logger.print_console('*** status: p1: %d / p2: %d ***' % (p1_ret, p2_ret)) @@ -185,17 +204,17 @@ def plan_next_operation(self, fmk_ops, dm, monitor, target, logger, fmk_feedback if p1_ret + p2_ret > 0: actions = [('SEPARATOR', UI(determinist=True)), - ('tSTRUCT', None, UI(deep=True)), - ('Cp', None, UI(idx=1)), ('Cp#1', None, UI(idx=3))] + ('tSTRUCT', UI(deep=True)), + ('Cp', UI(idx=1)), ('Cp#1', UI(idx=3))] elif -5 < p1_ret + p2_ret <= 0: - actions = ['SHAPE#specific', ('C#2', None, UI(path='.*prefix.$')), ('Cp#2', None, UI(idx=1))] + actions = ['SHAPE#specific', ('C#2', UI(path='.*prefix.$')), ('Cp#2', UI(idx=1))] else: actions = ['SHAPE#3', 'tTYPE#3'] - op.add_instruction(actions) + op.add_instruction(actions, tg_ids=[7,8]) if self.mode == 1: - actions_sup = ['SEPARATOR#2', ('tSTRUCT#2', None, UI(deep=True)), ('SIZE', None, UI(sz=10))] + actions_sup = ['SEPARATOR#2', ('tSTRUCT#2', UI(deep=True)), ('SIZE', UI(sz=10))] op.add_instruction(actions_sup) self.cpt += 1 @@ -210,7 +229,7 @@ def do_after_all(self, fmk_ops, dm, monitor, target, logger): self.detected_error += 1 linst.set_instruction(LastInstruction.RecordData) linst.set_operator_feedback('This input has crashed the target!') - + linst.set_operator_status(0) if self.cpt > self.max_steps and self.detected_error < 9: linst.set_operator_feedback("We have less than 9 data that trigger some problem with the target!" " You win!") diff --git a/test/__init__.py b/test/__init__.py index 5e72eb3..24bd0c0 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -57,9 +57,12 @@ help='Run all test cases. Some can take lot of time. (Disabled by default.)') parser.add_argument('--ignore-dm-specifics', action='store_true', help='Run Data Models specific test cases. (Enabled by default.)') +parser.add_argument('--exit-on-import-error', action='store_true', + help='Exit on Data Models or Projects import errors. (Disabled by default.)') test_args = parser.parse_known_args() run_long_tests = test_args[0].all ignore_data_model_specifics = test_args[0].ignore_dm_specifics +exit_on_import_error = test_args[0].exit_on_import_error args = [sys.argv[0]] + test_args[1] diff --git a/test/integration/test_integration.py b/test/integration/test_integration.py index 882b1bf..0fd6073 100644 --- a/test/integration/test_integration.py +++ b/test/integration/test_integration.py @@ -33,31 +33,28 @@ from framework.value_types import * -import data_models.example as example +import data_models.tutorial.example as example from framework.fuzzing_primitives import * from framework.plumbing import * from framework.data_model import * from framework.encoders import * - -from test import ignore_data_model_specifics, run_long_tests +from test import ignore_data_model_specifics, run_long_tests, exit_on_import_error def setUpModule(): global fmk, dm, results - fmk = FmkPlumbing() + fmk = FmkPlumbing(exit_on_error=exit_on_import_error, debug_mode=True) fmk.run_project(name='tuto', dm_name='example') dm = example.data_model results = collections.OrderedDict() + fmk.prj.reset_knowledge() def tearDownModule(): global fmk fmk.exit_fmk() -class TEST_Fuzzy_INT16(Fuzzy_INT16): - values = [0xDEAD, 0xBEEF, 0xCAFE] - ######## Tests cases begins Here ######## # Legacy --> Need to be revamped @@ -670,120 +667,6 @@ def test_01(self): for i in node_ex1.iter_paths(only_paths=True): print(i) - print('\n*** test 13: test typed_value Node') - - print('\n*** test 13.1:') - - tux = dm.get_atom('TUX') - - crit = NodeInternalsCriteria(mandatory_attrs=[NodeInternals.Mutable], - node_kinds=[NodeInternals_TypedValue]) - - print('> before changing types:') - - vt = {} - l1 = tux.get_reachable_nodes(internals_criteria=crit) - for e in l1: - print('Node.name: ', e.name, ', Id: ', e) - print('Node.env: ', e.env) - print('Node.value_type: ', e.cc.get_value_type()) - vt[e] = e.cc.get_value_type() - if issubclass(vt[e].__class__, VT_Alt): # isinstance(vt[e], (BitField, String)): - continue - compat = list(vt[e].compat_cls.values()) - compat.remove(vt[e].__class__) - print('Compatible types: ', compat) - - c = random.choice(compat) - # while True: - # c = random.choice(compat) - # if c != vt[e].__class__: - # break - - e.set_values(value_type=c()) - - res1 = False - if len(l1) == 29: - res1 = True - - print('\n> after changing types:') - - l1 = tux.get_reachable_nodes(internals_criteria=crit) - for e in l1: - print('Node.name', e.name) - print('Node.value_type:', e.cc.get_value_type()) - new_vt = e.cc.get_value_type() - # if isinstance(new_vt, BitField): - if issubclass(vt[e].__class__, VT_Alt): - continue - if new_vt.__class__ == vt[e].__class__: - res2 = False - break - else: - res2 = True - - # print(res1, res2) - # raise ValueError - - print('\n*** test 13.2:') - - evt = dm.get_atom('TVE') - l = evt.get_reachable_nodes(internals_criteria=crit) - - print('\n> part1:\n') - - print('--[ EVT1 ]-----[ EVT2 ]--') - for i in range(7): - print(evt.to_bytes()) - evt.unfreeze_all() - - vt = {} - for e in l: - print('-----') - print('Node.name: ', e.name) - print('Node.env: ', e.env) - print('Node.value_type: ', e.cc.get_value_type()) - vt[e] = e.cc.get_value_type() - # if isinstance(vt[e], BitField): - if issubclass(vt[e].__class__, VT_Alt): - continue - compat = list(vt[e].compat_cls.values()) - print('Compatible types: ', compat) - fuzzy = list(vt[e].fuzzy_cls.values()) - print('Fuzz compat types: ', fuzzy) - c = random.choice(fuzzy) - e.set_values(value_type=c()) - print("Set new value type '%s' for the Node %s!" % (c, e.name)) - - print('\n> part2:\n') - - print('--[ EVT1 ]-----[ EVT2 ]--') - for i in range(20): - s = evt.to_bytes() - print(s) - if evt.env.exhausted_node_exists(): - print("Exhausted Elts Exists!") - l = evt.env.get_exhausted_nodes() - for e in l: - evt.env.clear_exhausted_node(e) - print('Node %s has been cleared!' % e.name) - if e.is_value(): - continue - vt = e.cc.get_value_type() - if isinstance(vt, BitField): - continue - if vt: - compat = list(vt.compat_cls.values()) - compat.remove(vt.__class__) - c = random.choice(compat) - e.set_values(value_type=c()) - print("Set new value type '%s' for the Node %s!" % (c, e.name)) - - evt.unfreeze_all() - - print(res1, res2) - results['test13'] = res1 and res2 - print('\n### SUMMARY ###') for k, v in results.items(): @@ -866,10 +749,6 @@ def test_TypedNode_1(self): vt[e] = e.cc.get_value_type() if issubclass(vt[e].__class__, VT_Alt): continue - compat = list(vt[e].compat_cls.values()) - print(' Compatible types: ', compat) - fuzzy = list(vt[e].fuzzy_cls.values()) - print(' Fuzz compat types: ', fuzzy) print('') @@ -877,12 +756,13 @@ def test_TypedNode_1(self): evt.make_finite(all_conf=True, recursive=True) evt.make_determinist(all_conf=True, recursive=True) evt.show() + orig_rnode = evt.to_bytes() prev_path = None turn_nb_list = [] tn_consumer = TypedNodeDisruption() for rnode, node, orig_node_val, i in ModelWalker(evt, tn_consumer, make_determinist=True, max_steps=300): print('=======[ %d ]========' % i) - print(' orig: ', rnode.to_bytes()) + print(' orig: ', orig_rnode) print(' ----') if node != None: print(' fuzzed: ', rnode.to_bytes()) @@ -893,7 +773,7 @@ def test_TypedNode_1(self): print(' current fuzzed node: %s' % current_path) prev_path = current_path vt = node.cc.get_value_type() - print(' node value type: ', vt) + print(' node value type (changed by disruptor): ', vt) if issubclass(vt.__class__, VT_Alt): print(' |- node fuzzy mode: ', vt._fuzzy_mode) print(' node value type determinist: ', vt.determinist) @@ -911,7 +791,9 @@ def test_TypedNode_1(self): break print('\nTurn number when Node has changed: %r, number of test cases: %d' % (turn_nb_list, i)) - good_list = [1, 13, 23, 33, 43, 52, 61, 71, 81, 91, 103, 113, 123, 133, 143, 152, 162, 172, 182, 191, 200, 214, 229] + good_list = [1, 13, 25, 37, 49, 55, 61, 73, 85, 97, 109, 121, 133, 145, 157, 163, 175, 187, + 199, 208, 217, 233, 248] + msg = "If Fuzzy_.values have been modified in size, the good_list should be updated.\n" \ "If BitField are in random mode [currently put in determinist mode], the fuzzy_mode can produce more" \ " or less value depending on drawn value when .get_value() is called (if the drawn value is" \ @@ -1282,13 +1164,48 @@ def test_BitField_various_features(self): print('*** after extension') - bf.unfreeze() + bf.reset_state() bf.value_type.extend_right(vt2) bf.show() + extended_val = 3151759922 + extended_bytes = b'\xbb\xdc\n2' + vt = bf.value_type self.assertEqual(vt.subfield_limits, [3, 8, 15, 19, 22, 26, 30, 32]) - # self.assertEqual(vt.subfield_limits, [3, 8, 16, 20, 23, 27, 31, 33]) + self.assertEqual(vt.get_current_raw_val(), extended_val) + self.assertEqual(vt.get_current_value(), extended_bytes) + + print('\n -=[ .extend_left() method ]=- \n') + + # vt3 == vt2 + vt3 = BitField(subfield_sizes=[4, 3, 4, 4, 2], + subfield_values=[None, [3, 5], [15], [14], [2]], + subfield_val_extremums=[[8, 12], None, None, None, None], + padding=0, lsb_padding=False, endian=VT.BigEndian) + bf2 = Node('BF', vt=vt3) + bf2.make_determinist(all_conf=True, recursive=True) + bf2.set_env(Env()) + + print('*** before extension') + bf2.show() + + # vt4 == vt1 + vt4 = BitField(subfield_sizes=[3, 5, 7], + subfield_values=[[2, 1], None, [10, 120]], + subfield_val_extremums=[None, [6, 15], None], + padding=0, lsb_padding=True, endian=VT.BigEndian) + + print('*** after extension') + + bf2.reset_state() + bf2.value_type.extend_left(vt4) + bf2.show() + + self.assertEqual(bf2.value_type.subfield_limits, [3, 8, 15, 19, 22, 26, 30, 32]) + self.assertEqual(bf2.value_type.get_current_raw_val(), extended_val) + self.assertEqual(bf2.value_type.get_current_value(), extended_bytes) + print('\n -=[ .set_subfield() .get_subfield() methods ]=- \n') vt.set_subfield(idx=3, val=5) @@ -1562,8 +1479,8 @@ def test_basics(self): nt_data = data.get_clone() raw_vals = [ - b' [!] ++++++++++ [!] ::=:: [!] ', b' [!] ++++++++++ [!] ::?:: [!] ', + b' [!] ++++++++++ [!] ::=:: [!] ', b' [!] ++++++++++ [!] ::\xff:: [!] ', b' [!] ++++++++++ [!] ::\x00:: [!] ', b' [!] ++++++++++ [!] ::\x01:: [!] ', @@ -1582,10 +1499,10 @@ def test_basics(self): b' [!] ++++++++++ [!] ::AAA' + b'\"%s\"' * 400 + b'::AAA::AAA::AAA::>:: [!] ', b' [!] ++++++++++ [!] ::AAA' + b'\r\n' * 100 + b'::AAA::AAA::AAA::>:: [!] ', b' [!] ++++++++++ [!] ::../../../../../../etc/password::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::../../../../../../Windows/system.ini::AAA::AAA::AAA::>:: [!] ', + b' [!] ++++++++++ [!] ::..\\..\\..\\..\\..\\..\\Windows\\system.ini::AAA::AAA::AAA::>:: [!] ', b' [!] ++++++++++ [!] ::file%n%n%n%nname.txt::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::=:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::?:: [!] ', + b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::=:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\xff:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\x00:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\x01:: [!] ', @@ -1604,18 +1521,18 @@ def test_basics(self): b' [!] ++++++++++ [!] ::AAA' + b'\"%s\"' * 400 + b'::AAA::>:: [!] ', b' [!] ++++++++++ [!] ::AAA' + b'\r\n' * 100 + b'::AAA::>:: [!] ', b' [!] ++++++++++ [!] ::../../../../../../etc/password::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::../../../../../../Windows/system.ini::AAA::>:: [!] ', + b' [!] ++++++++++ [!] ::..\\..\\..\\..\\..\\..\\Windows\\system.ini::AAA::>:: [!] ', b' [!] ++++++++++ [!] ::file%n%n%n%nname.txt::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA::AAA::=:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::?:: [!] ', + b' [!] ++++++++++ [!] ::AAA::AAA::=:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::\xff:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::\x00:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::\x01:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::\x80:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::\x7f:: [!] ', - b' [!] >>>>>>>>>> [!] ::=:: [!] ', b' [!] >>>>>>>>>> [!] ::?:: [!] ', + b' [!] >>>>>>>>>> [!] ::=:: [!] ', b' [!] >>>>>>>>>> [!] ::\xff:: [!] ', b' [!] >>>>>>>>>> [!] ::\x00:: [!] ', b' [!] >>>>>>>>>> [!] ::\x01:: [!] ', @@ -1634,10 +1551,10 @@ def test_basics(self): b' [!] >>>>>>>>>> [!] ::AAA' + b'\"%s\"' * 400 + b'::AAA::AAA::AAA::>:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA' + b'\r\n' * 100 + b'::AAA::AAA::AAA::>:: [!] ', b' [!] >>>>>>>>>> [!] ::../../../../../../etc/password::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::../../../../../../Windows/system.ini::AAA::AAA::AAA::>:: [!] ', + b' [!] >>>>>>>>>> [!] ::..\\..\\..\\..\\..\\..\\Windows\\system.ini::AAA::AAA::AAA::>:: [!] ', b' [!] >>>>>>>>>> [!] ::file%n%n%n%nname.txt::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::=:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::?:: [!] ', + b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::=:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\xff:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\x00:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\x01:: [!] ', @@ -1656,10 +1573,10 @@ def test_basics(self): b' [!] >>>>>>>>>> [!] ::AAA' + b'\"%s\"' * 400 + b'::AAA::>:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA' + b'\r\n' * 100 + b'::AAA::>:: [!] ', b' [!] >>>>>>>>>> [!] ::../../../../../../etc/password::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::../../../../../../Windows/system.ini::AAA::>:: [!] ', + b' [!] >>>>>>>>>> [!] ::..\\..\\..\\..\\..\\..\\Windows\\system.ini::AAA::>:: [!] ', b' [!] >>>>>>>>>> [!] ::file%n%n%n%nname.txt::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA::AAA::=:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::?:: [!] ', + b' [!] >>>>>>>>>> [!] ::AAA::AAA::=:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::\xff:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::\x00:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::\x01:: [!] ', @@ -1667,7 +1584,7 @@ def test_basics(self): b' [!] >>>>>>>>>> [!] ::AAA::AAA::\x7f:: [!] ', ] - tn_consumer = TypedNodeDisruption(respect_order=True) + tn_consumer = TypedNodeDisruption(respect_order=True, ignore_separator=True) ic = NodeInternalsCriteria(mandatory_attrs=[NodeInternals.Mutable], negative_attrs=[NodeInternals.Separator], node_kinds=[NodeInternals_TypedValue], @@ -1709,7 +1626,7 @@ def test_TypedNodeDisruption_1(self): for rnode, consumed_node, orig_node_val, idx in ModelWalker(nt, tn_consumer, make_determinist=True, max_steps=300): print(colorize('[%d] ' % idx + repr(rnode.to_bytes()), rgb=Color.INFO)) - self.assertEqual(idx, 27) + self.assertEqual(idx, 21) def test_TypedNodeDisruption_2(self): nt = self.dm.get_atom('Simple') @@ -1733,7 +1650,7 @@ def test_TypedNodeDisruption_3(self): for rnode, consumed_node, orig_node_val, idx in ModelWalker(nt, tn_consumer, make_determinist=True, max_steps=-1): print(colorize('[%d] ' % idx + repr(rnode.to_bytes()), rgb=Color.INFO)) - self.assertEqual(idx, 450) + self.assertEqual(idx, 444) def test_TypedNodeDisruption_BitfieldCollapse(self): ''' @@ -1748,13 +1665,13 @@ def test_TypedNodeDisruption_BitfieldCollapse(self): # self.assertEqual(data['smscmd/TP-DCS'].to_bytes(), b'\xF6') corrupt_table = { - 1: b'\xF7', - 2: b'\xF4', - 3: b'\xF5', - 4: b'\xF2', - 5: b'\xFE', - 6: b'\x00', - 7: b'\xE0' + 1: b'\x06', + 2: b'\xE6', + 3: b'\x16', + 4: b'\xF7', + 5: b'\xF4', + 6: b'\xF5', + 7: b'\xF2' } tn_consumer = TypedNodeDisruption(max_runs_per_node=1) @@ -1771,8 +1688,6 @@ def test_TypedNodeDisruption_BitfieldCollapse(self): bin(struct.unpack('B', consumed_node.to_bytes())[0]))) print('result: {!s} ({!s})'.format(binascii.b2a_hex(rnode['smscmd/TP-DCS$'].to_bytes()), bin(struct.unpack('B', rnode['smscmd/TP-DCS$'].to_bytes())[0]))) - rnode.unfreeze(recursive=True, reevaluate_constraints=True) - rnode.freeze() rnode['smscmd/TP-DCS$'].show() self.assertEqual(rnode['smscmd/TP-DCS'].to_bytes(), corrupt_table[idx]) @@ -1833,7 +1748,7 @@ def test_JPG(self): print(colorize('number of imgs: %d' % idx, rgb=Color.INFO)) - self.assertEqual(idx, 116) + self.assertEqual(idx, 112) def test_USB(self): dm_usb = fmk.get_data_model_by_name('usb') @@ -1849,8 +1764,7 @@ def test_USB(self): print(colorize('number of confs: %d' % idx, rgb=Color.INFO)) - self.assertIn(idx, [527]) - + self.assertIn(idx, [542]) class TestNodeFeatures(unittest.TestCase): @@ -2432,30 +2346,31 @@ def test_collapse_padding(self): 'shape_type': MH.Ordered, 'custo_set': MH.Custo.NTerm.CollapsePadding, 'contents': [ - {'name': 'part1', - 'determinist': True, - 'contents': BitField(subfield_sizes=[3, 1], padding=0, endian=VT.BigEndian, - subfield_values=[None, [1]], - subfield_val_extremums=[[1, 3], None]) - }, {'name': 'sublevel', 'contents': [ - {'name': 'part2_o1', - 'exists_if': (BitFieldCondition(sf=0, val=[1]), 'part1'), - 'contents': BitField(subfield_sizes=[2, 2, 1], endian=VT.BigEndian, - subfield_values=[[1, 2], [3], [0]]) - }, - {'name': 'part2_o2', - 'exists_if': (BitFieldCondition(sf=0, val=[1]), 'part1'), + {'name': 'part2_msb', + 'exists_if': (BitFieldCondition(sf=0, val=[1]), 'part1_lsb'), 'contents': BitField(subfield_sizes=[2, 2], endian=VT.BigEndian, subfield_values=[[3], [3]]) }, + {'name': 'part2_middle', + 'exists_if': (BitFieldCondition(sf=0, val=[1]), 'part1_lsb'), + 'contents': BitField(subfield_sizes=[2, 2, 1], endian=VT.BigEndian, + subfield_values=[[1, 2], [3], [0]]) + }, {'name': 'part2_KO', - 'exists_if': (BitFieldCondition(sf=0, val=[2]), 'part1'), + 'exists_if': (BitFieldCondition(sf=0, val=[2]), 'part1_lsb'), 'contents': BitField(subfield_sizes=[2, 2], endian=VT.BigEndian, subfield_values=[[1], [1]]) } - ]} + ]}, + {'name': 'part1_lsb', + 'determinist': True, + 'contents': BitField(subfield_sizes=[3, 1], padding=0, endian=VT.BigEndian, + subfield_values=[None, [1]], + subfield_val_extremums=[[1, 3], None]) + }, + ]} mb = NodeBuilder() @@ -2469,9 +2384,128 @@ def test_collapse_padding(self): len(raw)) result = b'\xf6\xc8' + self.assertEqual(result, raw) + + + abs_test_desc = \ + {'name': 'test', + 'contents': [ + {'name': 'prefix', + 'contents': String(values=['prefix'])}, + {'name': 'TP-DCS', # Data Coding Scheme (refer to GSM 03.38) + 'custo_set': MH.Custo.NTerm.CollapsePadding, + 'contents': [ + {'name': '8-bit', + 'determinist': True, + 'contents': BitField(subfield_sizes=[8], endian=VT.BigEndian, + subfield_values=[ + [0xAA]], + ) }, + {'name': 'msb', + 'determinist': True, + 'contents': BitField(subfield_sizes=[4], endian=VT.BigEndian, + subfield_values=[ + [0b1111,0b1101,0b1100,0b0000]], + ) }, + {'name': 'lsb1', + 'determinist': True, + 'exists_if': (BitFieldCondition(sf=0, val=[0b1111]), 'msb'), + 'contents': BitField(subfield_sizes=[2,1,1,8], endian=VT.BigEndian, + subfield_values=[[0b10,0b11,0b00,0b01], + [1,0], + [0],[0xFE]] + ) }, + {'name': 'lsb2', + 'determinist': True, + 'exists_if': (BitFieldCondition(sf=0, val=[0b1101,0b1100]), 'msb'), + 'contents': BitField(subfield_sizes=[2,1,1], endian=VT.BigEndian, + subfield_values=[[0b10,0b11,0b00,0b01], + [0], + [0,1]] + ) }, + {'name': 'lsb31', + 'determinist': True, + 'exists_if': (BitFieldCondition(sf=0, val=[0]), 'msb'), + 'contents': BitField(subfield_sizes=[3], endian=VT.BigEndian, + subfield_values=[ + [0,4] + ] + ) }, + + {'name': 'lsb32', + 'determinist': True, + 'exists_if': (BitFieldCondition(sf=0, val=[0]), 'msb'), + 'contents': BitField(subfield_sizes=[8], endian=VT.BigEndian, + subfield_values=[ + [0,0x5c] + ] + ) }, + + {'name': 'lsb33', + 'determinist': True, + 'exists_if': (BitFieldCondition(sf=0, val=[0]), 'msb'), + 'contents': BitField(subfield_sizes=[1], endian=VT.BigEndian, + subfield_values=[ + [0,1] + ] + ) }, + ]}, + {'name': 'suffix', + 'contents': String(values=['suffix'])} + ]} + + mb = NodeBuilder() + node = mb.create_graph_from_desc(abs_test_desc) + node_abs = node.get_clone() + + raw = node.to_bytes() + node.show() # part2_KO should not be displayed + print(raw, binascii.b2a_hex(raw), + list(map(lambda x: bin(x), struct.unpack('>' + 'B' * len(raw), raw))), + len(raw)) + + result = b'prefix\xaa\xff\xe6suffix' + self.assertEqual(result, raw) + + print('\n*** Absorption test ***') + + result = b'prefix\xaa\xff\xe2suffix' + abs_result = node_abs.absorb(result) + print('\n--> Absorption status: {!r}\n'.format(abs_result)) + self.assertEqual(abs_result[0], AbsorbStatus.FullyAbsorbed) + raw = node_abs.to_bytes() + node_abs.show() # part2_KO should not be displayed + print(raw, binascii.b2a_hex(raw), + list(map(lambda x: bin(x), struct.unpack('>' + 'B' * len(raw), raw))), + len(raw)) + + self.assertEqual(result, raw) + + result = b'prefix\xaa\xdasuffix' + abs_result = node_abs.absorb(result) + print('\n--> Absorption status: {!r}\n'.format(abs_result)) + self.assertEqual(abs_result[0], AbsorbStatus.FullyAbsorbed) + raw = node_abs.to_bytes() + node_abs.show() # part2_KO should not be displayed + print(raw, binascii.b2a_hex(raw), + list(map(lambda x: bin(x), struct.unpack('>' + 'B' * len(raw), raw))), + len(raw)) + + self.assertEqual(result, raw) + + result = b'prefix\xaa\x08\xb9suffix' + abs_result = node_abs.absorb(result) + print('\n--> Absorption status: {!r}\n'.format(abs_result)) + self.assertEqual(abs_result[0], AbsorbStatus.FullyAbsorbed) + raw = node_abs.to_bytes() + node_abs.show() # part2_KO should not be displayed + print(raw, binascii.b2a_hex(raw), + list(map(lambda x: bin(x), struct.unpack('>' + 'B' * len(raw), raw))), + len(raw)) self.assertEqual(result, raw) + def test_search_primitive(self): data = fmk.dm.get_external_atom(dm_name='mydf', data_id='exist_cond') @@ -3056,7 +3090,7 @@ def test_data_makers(self): except: print("\n*** WARNING: Data Model '{:s}' not tested because" \ " the loading process has failed ***\n".format(dm.name)) - continue + raise print("Test '%s' Data Model" % dm.name) for data_id in dm.atom_identifiers(): @@ -3066,6 +3100,22 @@ def test_data_makers(self): # data.show(raw_limit=200) print('Success!') + @unittest.skipIf(not run_long_tests, "Long test case") + def test_data_model_specifics(self): + + for dm in fmk.dm_list: + try: + dm.load_data_model(fmk._name2dm) + except: + print("\n*** WARNING: Data Model '{:s}' not tested because" \ + " the loading process has failed ***\n".format(dm.name)) + raise + + print("Validating '{:s}' Data Model".format(dm.name)) + + ok = dm.validation_tests() + self.assertTrue(ok) + def test_generic_generators(self): dm = fmk.get_data_model_by_name('mydf') dm.load_data_model(fmk._name2dm) @@ -3291,6 +3341,10 @@ def test_zip_specifics(self): @ddt.ddt class TestDataModelHelpers(unittest.TestCase): + @classmethod + def setUpClass(cls): + fmk.run_project(name='tuto', tg_ids=0, dm_name='mydf') + @ddt.data("HTTP_version_regex", ("HTTP_version_regex", 17), ("HTTP_version_regex", "whatever")) def test_regex(self, regex_node_name): HTTP_version_classic = \ @@ -3349,14 +3403,69 @@ def test_regex_shape(self, regexp, shapes): self.assertEqual(len(shapes), 0) + def test_xml_helpers(self): + + xml5_samples = [ + '\n' + '\n\n\n0\n\n\nMyUser' + '\n\n\nplopi\n\n\n', + '\n ' + '\n\t \n\n56\n\t\n\n\nMyUser' + '\n\n\nohohoh! \n\n\n'] + + + for idx, sample in enumerate(xml5_samples): + xml_atom = fmk.dm.get_atom('xml5') + status, off, size, name = xml_atom.absorb(sample, constraints=AbsFullCsts()) + + print('{:s} Absorb Status: {:d}, {:d}, {:s}'.format(status, off, size, name)) + print(' \_ length of original data: {:d}'.format(len(sample))) + print(' \_ remaining: {!r}'.format(sample[size:size+1000])) + + xml_atom.show() + assert status == AbsorbStatus.FullyAbsorbed + + data_sizes = [211, 149, 184] + for i in range(100): + data = fmk.get_data(['XML5', ('tWALK', UI(path='xml5/command/start-tag/content/attr1/cmd_val'))]) + if data is None: + break + assert len(data.to_bytes()) == data_sizes[i] + go_on = fmk.send_data_and_log([data]) + if not go_on: + raise ValueError + else: + raise ValueError + + assert i == 3 + + specific_cases_checked = False + for i in range(100): + data = fmk.get_data(['XML5', ('tTYPE', UI(path='xml5/command/LOGIN/start-tag/content/attr1/val'))]) + if data is None: + break + node_to_check = data.content['xml5/command/LOGIN/start-tag/content/attr1/val'] + if node_to_check.to_bytes() == b'None': + # one case should trigger this condition + specific_cases_checked = True + go_on = fmk.send_data_and_log([data]) + if not go_on: + raise ValueError + else: + raise ValueError + + assert i == 22, 'number of test cases: {:d}'.format(i) + assert specific_cases_checked class TestFMK(unittest.TestCase): @classmethod def setUpClass(cls): - fmk.run_project(name='tuto', dm_name='mydf', tg=0) + fmk.run_project(name='tuto', tg_ids=0, dm_name='mydf') + fmk.prj.reset_target_mappings() def setUp(self): - fmk.reload_all(tg_num=0) + fmk.reload_all(tg_ids=[0]) + fmk.prj.reset_target_mappings() def test_generic_disruptors_01(self): dmaker_type = 'TESTNODE' @@ -3372,7 +3481,7 @@ def test_generic_disruptors_01(self): print("\n\n---[ Tested Disruptor %r ]---" % dis) if dis == 'EXT': - act = [dmaker_type, (dis, None, UI(cmd='/bin/cat', file_mode=True))] + act = [dmaker_type, (dis, UI(cmd='/bin/cat', file_mode=True))] d = fmk.get_data(act) else: act = [dmaker_type, dis] @@ -3392,7 +3501,7 @@ def test_separator_disruptor(self): d = fmk.get_data(['SEPARATOR', 'tSEP']) if d is None: break - fmk.new_transfer_preamble() + fmk._setup_new_sending() fmk._log_data(d) self.assertGreater(i, 2) @@ -3408,14 +3517,14 @@ def test_struct_disruptor(self): outcomes = [] - act = [('EXIST_COND', UI(determinist=True)), 'tWALK', ('tSTRUCT', None)] + act = [('EXIST_COND', UI(determinist=True)), 'tWALK', 'tSTRUCT'] for i in range(4): for j in range(10): d = fmk.get_data(act) if d is None: print('--> Exiting (need new input)') break - fmk.new_transfer_preamble() + fmk._setup_new_sending() fmk._log_data(d) outcomes.append(d.to_bytes()) d.show() @@ -3430,13 +3539,13 @@ def test_struct_disruptor(self): expected_idx = 10 idx = 0 - act = [('SEPARATOR', UI(determinist=True)), ('tSTRUCT', None, UI(deep=True))] + act = [('SEPARATOR', UI(determinist=True)), ('tSTRUCT', UI(deep=True))] for j in range(10): d = fmk.get_data(act) if d is None: print('--> Exiting (need new input)') break - fmk.new_transfer_preamble() + fmk._setup_new_sending() fmk._log_data(d) outcomes.append(d.to_bytes()) d.show() @@ -3458,7 +3567,7 @@ def test_typednode_disruptor(self): if d is None: print('--> Exiting (need new input)') break - fmk.new_transfer_preamble() + fmk._setup_new_sending() fmk._log_data(d) outcomes.append(d.to_bytes()) d.show() @@ -3468,12 +3577,15 @@ def test_typednode_disruptor(self): def test_operator_1(self): - fmk.launch_operator('MyOp', user_input=UserInputContainer(specific=UI(max_steps=100, mode=1))) - print('\n*** Last data ID: {:d}'.format(fmk.lg.last_data_id)) + fmk.reload_all(tg_ids=[7,8]) + + fmk.launch_operator('MyOp', user_input=UI(max_steps=100, mode=1)) + last_data_id = max(fmk.lg._last_data_IDs.values()) + print('\n*** Last data ID: {:d}'.format(last_data_id)) fmkinfo = fmk.fmkDB.execute_sql_statement( "SELECT CONTENT FROM FMKINFO " "WHERE DATA_ID == {data_id:d} " - "ORDER BY ERROR DESC;".format(data_id=fmk.lg.last_data_id) + "ORDER BY ERROR DESC;".format(data_id=last_data_id) ) self.assertTrue(fmkinfo) for info in fmkinfo: @@ -3485,26 +3597,28 @@ def test_operator_1(self): @unittest.skipIf(not run_long_tests, "Long test case") def test_operator_2(self): + fmk.reload_all(tg_ids=[7,8]) + + myop = fmk.get_operator(name='MyOp') fmk.launch_operator('MyOp') - fbk = fmk.fmkDB.last_feedback["Operator 'MyOp'"][0]['content'] + + fbk = fmk.feedback_gate.get_feedback_from(myop)[0]['content'] print(fbk) self.assertIn(b'You win!', fbk) fmk.launch_operator('MyOp') - fbk = fmk.fmkDB.last_feedback["Operator 'MyOp'"][0]['content'] + fbk = fmk.feedback_gate.get_feedback_from(myop)[0]['content'] print(fbk) self.assertIn(b'You loose!', fbk) - def test_scenario_infra_01(self): + def test_scenario_infra_01a(self): - print('\n*** test scenario SC_NO_REGEN') + print('\n*** test scenario SC_NO_REGEN via _send_data()') base_qty = 0 for i in range(100): data = fmk.get_data(['SC_NO_REGEN']) data_list = fmk._send_data([data]) # needed to make the scenario progress - # send_data_and_log() should be used for more complex scenarios - # hooking the framework in more places. if not data_list: base_qty = i break @@ -3519,7 +3633,7 @@ def test_scenario_infra_01(self): 'DPHandOver', 'NoMoreData']) self.assertEqual(base_qty, 55) - print('\n*** test scenario SC_AUTO_REGEN') + print('\n*** test scenario SC_AUTO_REGEN via _send_data()') for i in range(base_qty * 3): data = fmk.get_data(['SC_AUTO_REGEN']) @@ -3527,10 +3641,48 @@ def test_scenario_infra_01(self): if not data_list: raise ValueError + @unittest.skipIf(not run_long_tests, "Long test case") + def test_scenario_infra_01b(self): + + print('\n*** test scenario SC_NO_REGEN via send_data_and_log()') + # send_data_and_log() is used to stimulate the framework in more places. + + base_qty = 0 + for i in range(100): + data = fmk.get_data(['SC_NO_REGEN']) + go_on = fmk.send_data_and_log([data]) + if not go_on: + base_qty = i + break + else: + raise ValueError + + err_list = fmk.get_error() + code_vector = [str(e) for e in err_list] + full_code_vector = [(str(e), e.msg) for e in err_list] + print('\n*** Retrieved error code vector: {!r}'.format(full_code_vector)) + + + self.assertEqual(code_vector, ['DataUnusable', 'HandOver', 'DataUnusable', 'HandOver', + 'DPHandOver', 'NoMoreData']) + self.assertEqual(base_qty, 55) + + print('\n*** test scenario SC_AUTO_REGEN via send_data_and_log()') + + for i in range(base_qty * 3): + data = fmk.get_data(['SC_AUTO_REGEN']) + go_on = fmk.send_data_and_log([data]) + if not go_on: + raise ValueError + + @unittest.skipIf(not run_long_tests, "Long test case") def test_scenario_infra_02(self): - fmk.reload_all(tg_num=1) # to collect feedback from monitoring probes + fmk.reload_all(tg_ids=[1]) # to collect feedback from monitoring probes + fmk.prj.reset_target_mappings() + fmk.prj.map_targets_to_scenario('ex1', {0: 1, 1: 1, None: 1}) + fmk.prj.map_targets_to_scenario('ex2', {0: 1, 1: 1, None: 1}) print('\n*** Test scenario EX1') diff --git a/test/unit/test_node_builder.py b/test/unit/test_node_builder.py index 5f2b30e..a83b9d0 100644 --- a/test/unit/test_node_builder.py +++ b/test/unit/test_node_builder.py @@ -113,7 +113,8 @@ def test_quantifiers(self, test_case): {'regex': "hi\x58", 'nodes': [{"values": ["hi\x58"]}]}, {'regex': "hi\x00hola", 'nodes': [{"values": ["hi\x00hola"]}]}, {'regex': "\xFFdom", 'nodes': [{"values": ["\xFFdom"]}]}, - {'regex': "\ddom", 'nodes': [{"alphabet": "0123456789"}, {"values": ["dom"]}]}, + {'regex': "\ddom", + 'nodes': [{"values": [i for i in range(0,10)], "type": vt.INT_str}, {"values": ["dom"]}]}, {'regex': "dom[abcd\d]", 'nodes': [{"values": ["dom"]}, {"alphabet": "abcd0123456789"}]}, {'regex': "[abcd]\x43", 'nodes': [{"alphabet": "abcd"}, {"values": ["\x43"]}]}, {'regex': "(abcd)\x53", 'nodes': [{"values": ["abcd"]}, {"values": ["\x53"]}]}, @@ -127,7 +128,7 @@ def test_quantifiers(self, test_case): 'nodes': [ {"type": fvt.INT_str, "values": [333,444]}, {"values": [u"foo-bar"]}, - {"alphabet": "0123456789"}, + {"values": [i for i in range(0,10)], "type": vt.INT_str}, {"alphabet": "th|is"}]}, {'regex': u"(333|444)|foo-bar|\||[th|is]", 'nodes': [ @@ -266,19 +267,19 @@ def test_types_recognition(self, test_case): {'regex': "[a-e]", 'nodes': [{"alphabet": "abcde"}]}, {'regex': "[a-ewxy]", 'nodes': [{"alphabet": "abcdewxy"}]}, - {'regex': "[1-9]", 'nodes': [{"alphabet": "123456789"}]}, + {'regex': "[1-9]", 'nodes': [{"values": [i for i in range(1,10)], 'type': vt.INT_str}]}, {'regex': "[what1-9]", 'nodes': [{"alphabet": "what123456789"}]}, {'regex': "[a-c1-9]", 'nodes': [{"alphabet": "abc123456789"}]}, {'regex': "[a-c1-9fin]", 'nodes': [{"alphabet": "abc123456789fin"}]}, {'regex': "[a-c9-9fin]", 'nodes': [{"alphabet": "abc9fin"}]}, {'regex': "[pa-cwho1-9fin]", 'nodes': [{"alphabet": "pabcwho123456789fin"}]}, - {'regex': "[\x33]", 'nodes': [{"alphabet": "\x33"}]}, - {'regex': "[\x33-\x35]", 'nodes': [{"alphabet": "\x33\x34\x35"}]}, + {'regex': "[\x33]", 'nodes': [{"values": [3], 'type': vt.INT_str}]}, + {'regex': "[\x33-\x35]", 'nodes': [{"values": [3,4,5], 'type': vt.INT_str}]}, {'regex': "[e\x33-\x35a]", 'nodes': [{"alphabet": "e\x33\x34\x35a"}]}, {'regex': u"[\u0033]", "charset": MH.Charset.UNICODE, - 'nodes': [{"alphabet": u"\u0033"}]}, + 'nodes': [{"values": [3], 'type': vt.INT_str}]}, {'regex': u"[\u0003-\u0005]", "charset": MH.Charset.UNICODE, 'nodes': [{"alphabet": u"\u0003\u0004\u0005"}]}, {'regex': u"[\u0333-\u0335]", "charset": MH.Charset.UNICODE, @@ -372,7 +373,7 @@ def assert_regex_is_valid(self, test_case): self._parser._create_terminal_node.assert_has_calls(calls) - self.assertEquals(self._parser._create_terminal_node.call_count, len(test_case['nodes'])) + self.assertEqual(self._parser._create_terminal_node.call_count, len(test_case['nodes'])) def assert_regex_is_invalid(self, test_case): diff --git a/tools/fmkdb.py b/tools/fmkdb.py index 0371558..7257e76 100755 --- a/tools/fmkdb.py +++ b/tools/fmkdb.py @@ -60,8 +60,10 @@ group.add_argument('-s', '--all-stats', action='store_true', help='Show all statistics') group = parser.add_argument_group('Fuddly Database Information') -group.add_argument('-i', '--info', type=int, metavar='DATA_ID', - help='Display information on the specified data ID') +group.add_argument('-i', '--data-id', type=int, metavar='DATA_ID', + help='Provide the data ID on which actions will be performed. Without ' + 'any other parameters the default action is to display ' + 'information on the specified data ID.') group.add_argument('--info-by-date', nargs=2, metavar=('START','END'), help='''Display information on data sent between START and END ''' '''(date format 'Year/Month/Day' or 'Year/Month/Day-Hour' or @@ -70,10 +72,12 @@ help='''Display information on all the data included within the specified data ID range''') -group.add_argument('--with-fbk', action='store_true', help='Display full feedback (expect --info)') -group.add_argument('--with-data', action='store_true', help='Display data content (expect --info)') +group.add_argument('--with-fbk', action='store_true', help='Display full feedback (expect --data-id)') +group.add_argument('--with-data', action='store_true', help='Display data content (expect --data-id)') group.add_argument('--without-fmkinfo', action='store_true', - help='Do not display fmkinfo (expect --info)') + help='Do not display fmkinfo (expect --data-id)') +group.add_argument('--without-analysis', action='store_true', + help='Do not display user analysis (expect --data-id)') group.add_argument('--limit', type=int, default=None, help='Limit the size of what is displayed from the sent data and the ' 'retrieved feedback (expect --with-data or --with-fbk).') @@ -91,13 +95,23 @@ group = parser.add_argument_group('Fuddly Database Analysis') group.add_argument('--data-with-impact', action='store_true', - help="Retrieve data that negatively impacted a target") + help="Retrieve data that negatively impacted a target. Analysis is performed " + "based on feedback status and user analysis if present") +group.add_argument('--data-with-impact-raw', action='store_true', + help="Retrieve data that negatively impacted a target. Analysis is performed " + "based on feedback status") group.add_argument('--data-without-fbk', action='store_true', help="Retrieve data without feedback") group.add_argument('--data-with-specific-fbk', metavar='FEEDBACK_REGEXP', help="Retrieve data with specific feedback provided as a regexp") - - +group.add_argument('-a', '--add-analysis', nargs=2, metavar=('IMPACT', 'COMMENT'), + help='''Add an impact analysis to a specific data ID (expect --data-id). + IMPACT should be either 0 (no impact) or 1 (impact), and COMMENT + provide information''') +group.add_argument('--disprove-impact', nargs=2, metavar=('FIRST_ID', 'LAST_ID'), type=int, + help='''Disprove the impact of a group of data present in the outcomes of + '--data-with-impact-raw'. The group is determined by providing the smaller data ID + (FIRST_ID) and the bigger data ID (LAST_ID).''') def handle_confirmation(): try: @@ -149,13 +163,14 @@ def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): display_stats = args.all_stats - data_info = args.info + data_ID = args.data_id data_info_by_date = args.info_by_date data_info_by_range = args.info_by_ids prj_name = args.project with_fbk = args.with_fbk with_data = args.with_data without_fmkinfo = args.without_fmkinfo + without_analysis = args.without_analysis limit_data_sz = args.limit raw_data = args.raw @@ -165,9 +180,12 @@ def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): remove_one_data = args.remove_one_data impact_analysis = args.data_with_impact + raw_impact_analysis = args.data_with_impact_raw data_without_fbk = args.data_without_fbk fbk_src = args.fbk_src data_with_specific_fbk = args.data_with_specific_fbk + add_analysis = args.add_analysis + disprove_impact = args.disprove_impact fmkdb = Database(fmkdb_path=fmkdb) ok = fmkdb.start() @@ -176,14 +194,48 @@ def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): rgb=Color.ERROR)) sys.exit(-1) + now = datetime.now() + if display_stats: fmkdb.display_stats(colorized=colorized) - elif data_info is not None: + elif add_analysis is not None: + try: + ia_impact = int(add_analysis[0]) + except ValueError: + print('*** IMPACT argument is incorrect! ***') + else: + ia_comment = add_analysis[1] + fmkdb.insert_analysis(data_ID, ia_comment, now, impact=bool(ia_impact)) + + + elif disprove_impact is not None: + + first_data_id = disprove_impact[0] + last_data_id = disprove_impact[1] + + data_list = fmkdb.get_data_with_impact(prj_name=prj_name, fbk_src=fbk_src, display=False, + raw_analysis=True) + data_list = sorted(data_list) + + if first_data_id not in data_list or last_data_id not in data_list: + print('*** Error with provided data IDs! ***') + else: + idx_first = data_list.index(first_data_id) + idx_last = data_list.index(last_data_id) + data_list_to_disprove = data_list[idx_first:idx_last + 1] + + for data_id in data_list_to_disprove: + fmkdb.insert_analysis(data_id, "Impact is disproved by user analysis. (False Positive.)", + now, impact=False) + + elif data_ID is not None: - fmkdb.display_data_info(data_info, with_data=with_data, with_fbk=with_fbk, - with_fmkinfo=not without_fmkinfo, fbk_src=fbk_src, + fmkdb.display_data_info(data_ID, with_data=with_data, with_fbk=with_fbk, + with_fmkinfo=not without_fmkinfo, + with_analysis=not without_analysis, + fbk_src=fbk_src, limit_data_sz=limit_data_sz, raw=raw_data, page_width=page_width, colorized=colorized) @@ -224,8 +276,9 @@ def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): else: fmkdb.remove_data(remove_one_data, colorized=colorized) - elif impact_analysis: + elif impact_analysis or raw_impact_analysis: fmkdb.get_data_with_impact(prj_name=prj_name, fbk_src=fbk_src, verbose=verbose, + raw_analysis=raw_impact_analysis, colorized=colorized) elif data_without_fbk: