From 0d031c8a558a70664b7ce1e7f580472cff73c2b8 Mon Sep 17 00:00:00 2001 From: Duncan Ogilvie Date: Sat, 6 Jan 2024 18:00:33 +0100 Subject: [PATCH 1/5] Update .gitignore to ignore virtual envs --- .gitignore | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6fc6b959b..34b943827 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,18 @@ # Build directory /build/* +dist/ +sdists/ # Emacs files *~ # Compiled python files +__pycache__/ *.py[cod] # Generated files *.egg* **.dot **.so -VERSION \ No newline at end of file +VERSION +# Virtual environments +venv*/ +.env/ +.venv*/ \ No newline at end of file From f2335970fea3670c261c162e65e72ebef2a4de2e Mon Sep 17 00:00:00 2001 From: Duncan Ogilvie Date: Sat, 6 Jan 2024 18:36:38 +0100 Subject: [PATCH 2/5] Remove trailing whitespace --- example/jitter/unpack_generic.py | 2 +- miasm/arch/aarch64/sem.py | 10 +++++----- miasm/expression/simplifications_common.py | 6 +++--- miasm/jitter/loader/pe.py | 18 +++++++++--------- miasm/os_dep/win_api_x86_32.py | 4 ++-- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/example/jitter/unpack_generic.py b/example/jitter/unpack_generic.py index 3329d2a93..e389a4b5f 100644 --- a/example/jitter/unpack_generic.py +++ b/example/jitter/unpack_generic.py @@ -35,7 +35,7 @@ def stop(jitter): if options.oep: # Set callbacks sb.jitter.add_breakpoint(int(options.oep, 0), stop) - + # Run until an error is encountered - IT IS UNLIKELY THE ORIGINAL ENTRY POINT try: sb.run() diff --git a/miasm/arch/aarch64/sem.py b/miasm/arch/aarch64/sem.py index 8cbab90ba..eaa012286 100644 --- a/miasm/arch/aarch64/sem.py +++ b/miasm/arch/aarch64/sem.py @@ -179,7 +179,7 @@ (3, 0, 2, 3, 0): APGAKeyLo_EL1, (3, 0, 2, 3, 1): APGAKeyHi_EL1, - + (3, 0, 4, 1, 0): SP_EL0, (3, 0, 4, 6, 0): ICC_PMR_EL1, # Alias ICV_PMR_EL1 @@ -285,7 +285,7 @@ (3, 0, 0, 0, 1): CTR_EL0, (3, 3, 0, 0, 7): DCZID_EL0, - + (3, 3, 4, 4, 0): FPCR, (3, 3, 4, 4, 1): FPSR, @@ -1578,13 +1578,13 @@ def msr(ir, instr, arg1, arg2, arg3, arg4, arg5, arg6): e.append(ExprAssign(zf, arg6[30:31])) e.append(ExprAssign(cf, arg6[29:30])) e.append(ExprAssign(of, arg6[28:29])) - + elif arg1.is_int(3) and arg2.is_int(3) and arg3.is_id("c4") and arg4.is_id("c2") and arg5.is_int(7): e.append(ExprAssign(tco, arg6[25:26])) elif arg1.is_int(3) and arg2.is_int(3) and arg3.is_id("c4") and arg4.is_id("c2") and arg5.is_int(0): e.append(ExprAssign(dit, arg6[24:25])) - + elif arg1.is_int(3) and arg2.is_int(0) and arg3.is_id("c4") and arg4.is_id("c2") and arg5.is_int(4): e.append(ExprAssign(uao, arg6[23:24])) @@ -1599,7 +1599,7 @@ def msr(ir, instr, arg1, arg2, arg3, arg4, arg5, arg6): e.append(ExprAssign(af, arg6[8:9])) e.append(ExprAssign(iff, arg6[7:8])) e.append(ExprAssign(ff, arg6[6:7])) - + elif arg1.is_int(3) and arg2.is_int(0) and arg3.is_id("c4") and arg4.is_id("c2") and arg5.is_int(2): e.append(ExprAssign(cur_el, arg6[2:4])) diff --git a/miasm/expression/simplifications_common.py b/miasm/expression/simplifications_common.py index 835f87231..a1dce84e9 100644 --- a/miasm/expression/simplifications_common.py +++ b/miasm/expression/simplifications_common.py @@ -1146,7 +1146,7 @@ def simp_cmp_bijective_op(expr_simp, expr): # a + b + c == a + b if not args_b: return ExprOp(TOK_EQUAL, ExprOp(op, *args_a), ExprInt(0, args_a[0].size)) - + arg_a = ExprOp(op, *args_a) arg_b = ExprOp(op, *args_b) return ExprOp(TOK_EQUAL, arg_a, arg_b) @@ -1782,7 +1782,7 @@ def simp_bcdadd_cf(_, expr): for i in range(0,16,4): nib_1 = (arg1.arg >> i) & (0xF) nib_2 = (arg2.arg >> i) & (0xF) - + j = (carry + nib_1 + nib_2) if (j >= 10): carry = 1 @@ -1807,7 +1807,7 @@ def simp_bcdadd(_, expr): for i in range(0,16,4): nib_1 = (arg1.arg >> i) & (0xF) nib_2 = (arg2.arg >> i) & (0xF) - + j = (carry + nib_1 + nib_2) if (j >= 10): carry = 1 diff --git a/miasm/jitter/loader/pe.py b/miasm/jitter/loader/pe.py index c988fc597..9af068e48 100644 --- a/miasm/jitter/loader/pe.py +++ b/miasm/jitter/loader/pe.py @@ -23,12 +23,12 @@ def get_pe_dependencies(pe_obj): """Collect the shared libraries upon which this PE depends. - + @pe_obj: pe object Returns a set of strings of DLL names. - + Example: - + pe = miasm.analysis.binary.Container.from_string(buf) deps = miasm.jitter.loader.pe.get_pe_dependencies(pe.executable) assert sorted(deps)[0] == 'api-ms-win-core-appcompat-l1-1-0.dll' @@ -63,12 +63,12 @@ def get_import_address_pe(e): """Compute the addresses of imported symbols. @e: pe object Returns a dict mapping from tuple (dll name string, symbol name string) to set of virtual addresses. - + Example: - + pe = miasm.analysis.binary.Container.from_string(buf) imports = miasm.jitter.loader.pe.get_import_address_pe(pe.executable) - assert imports[('api-ms-win-core-rtlsupport-l1-1-0.dll', 'RtlCaptureStackBackTrace')] == {0x6b88a6d0} + assert imports[('api-ms-win-core-rtlsupport-l1-1-0.dll', 'RtlCaptureStackBackTrace')] == {0x6b88a6d0} """ import2addr = defaultdict(set) if e.DirImport.impdesc is None: @@ -732,7 +732,7 @@ def format_func_info(self, func_info, func_addr): "entry_module_addr": func_addr, "entry_memory_addr": self.cur_address, } - + def transition(self, data): if self.state == self.STATE_SEARCH: if data in self.func_addrs: @@ -760,7 +760,7 @@ def transition(self, data): self.transition(data) else: raise ValueError() - + def run(self): while True: data, address = yield @@ -804,7 +804,7 @@ def recover_import(self, update_libs=True, align_hypothesis=False): @update_libs: if set (default), update `libs` object with founded addresses @align_hypothesis: if not set (default), do not consider import addresses are written on aligned addresses - + Return the list of candidates """ candidates = [] diff --git a/miasm/os_dep/win_api_x86_32.py b/miasm/os_dep/win_api_x86_32.py index e9c5fd4a9..b60ff879f 100644 --- a/miasm/os_dep/win_api_x86_32.py +++ b/miasm/os_dep/win_api_x86_32.py @@ -2452,7 +2452,7 @@ def user32_GetKeyboardType(jitter): jitter.func_ret_stdcall(ret_ad, ret) - + class startupinfo(object): """ typedef struct _STARTUPINFOA { @@ -2528,7 +2528,7 @@ def kernel32_GetStartupInfo(jitter, funcname, set_str): Retrieves the contents of the STARTUPINFO structure that was specified when the calling process was created. - + https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getstartupinfow """ From 0b0de2cbd1a5949cd8819899169446c5cf524cc3 Mon Sep 17 00:00:00 2001 From: Duncan Ogilvie Date: Sat, 6 Jan 2024 18:02:38 +0100 Subject: [PATCH 3/5] Replace /!\ with [!] to fix a few warnings miasm/expression/expression_helper.py:90: SyntaxWarning: invalid escape sequence '\ ' --- miasm/analysis/debugging.py | 2 +- miasm/analysis/dse.py | 4 ++-- miasm/expression/expression.py | 4 ++-- miasm/expression/expression_helper.py | 4 ++-- miasm/expression/simplifications_common.py | 2 +- miasm/jitter/loader/utils.py | 2 +- miasm/loader/minidump.py | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/miasm/analysis/debugging.py b/miasm/analysis/debugging.py index f114d901a..d5f59d492 100644 --- a/miasm/analysis/debugging.py +++ b/miasm/analysis/debugging.py @@ -377,7 +377,7 @@ def do_display(self, arg): args = arg.split(" ") if args[-1].lower() not in ["on", "off"]: - self.print_warning("/!\ %s not in 'on' / 'off'" % args[-1]) + self.print_warning("[!] %s not in 'on' / 'off'" % args[-1]) return mode = args[-1].lower() == "on" d = {} diff --git a/miasm/analysis/dse.py b/miasm/analysis/dse.py index 5e6c4e8d9..116747341 100644 --- a/miasm/analysis/dse.py +++ b/miasm/analysis/dse.py @@ -234,7 +234,7 @@ def breakpoint(self, jitter): def handle(self, cur_addr): r"""Handle destination @cur_addr: Expr of the next address in concrete execution - /!\ cur_addr may be a loc_key + [!] cur_addr may be a loc_key In this method, self.symb is in the "just before branching" state """ @@ -475,7 +475,7 @@ def update_state_from_concrete(self, cpu=True, mem=False): @cpu: (optional) if set, update registers' value @mem: (optional) if set, update memory value - /!\ all current states will be loss. + [!] all current states will be loss. This function is usually called when states are no more synchronized (at the beginning, returning from an unstubbed syscall, ...) """ diff --git a/miasm/expression/expression.py b/miasm/expression/expression.py index c507f19f6..e5debb341 100644 --- a/miasm/expression/expression.py +++ b/miasm/expression/expression.py @@ -2146,7 +2146,7 @@ def expr_is_sNaN(expr): def expr_is_float_lower(op1, op2): """Return 1 on 1 bit if @op1 < @op2, 0 otherwise. - /!\ Assume @op1 and @op2 are not NaN + [!] Assume @op1 and @op2 are not NaN Comparison is the floating point one, defined in IEEE754 """ sign1, sign2 = op1.msb(), op2.msb() @@ -2160,7 +2160,7 @@ def expr_is_float_lower(op1, op2): def expr_is_float_equal(op1, op2): """Return 1 on 1 bit if @op1 == @op2, 0 otherwise. - /!\ Assume @op1 and @op2 are not NaN + [!] Assume @op1 and @op2 are not NaN Comparison is the floating point one, defined in IEEE754 """ sign1, sign2 = op1.msb(), op2.msb() diff --git a/miasm/expression/expression_helper.py b/miasm/expression/expression_helper.py index 299e52e6c..5bd2276db 100644 --- a/miasm/expression/expression_helper.py +++ b/miasm/expression/expression_helper.py @@ -89,7 +89,7 @@ def merge_sliceto_slice(expr): def is_pure_int(e): """ return True if expr is only composed with integers - /!\ ExprCond returns True is src1 and src2 are integers + [!] ExprCond returns True is src1 and src2 are integers """ def modify_cond(e): if isinstance(e, m2_expr.ExprCond): @@ -444,7 +444,7 @@ def _gen(cls, size=32, depth=1): """Internal function for generating sub-expression according to options @size: (optional) Operation size @depth: (optional) Expression depth - /!\ @generated_elements is left modified + [!] @generated_elements is left modified """ # Perfect tree handling if not cls.perfect_tree: diff --git a/miasm/expression/simplifications_common.py b/miasm/expression/simplifications_common.py index a1dce84e9..9156ee671 100644 --- a/miasm/expression/simplifications_common.py +++ b/miasm/expression/simplifications_common.py @@ -1275,7 +1275,7 @@ def simp_cond_eq_zero(_, expr): def simp_sign_inf_zeroext(expr_s, expr): """ - /!\ Ensure before: X.zeroExt(X.size) => X + [!] Ensure before: X.zeroExt(X.size) => X X.zeroExt() 0 X.zeroExt() <=s 0 => X == 0 diff --git a/miasm/jitter/loader/utils.py b/miasm/jitter/loader/utils.py index 73809141f..7f913d765 100644 --- a/miasm/jitter/loader/utils.py +++ b/miasm/jitter/loader/utils.py @@ -65,7 +65,7 @@ def lib_get_add_func(self, libad, imp_ord_or_name, dst_ad=None): # imp_ord_or_name = vm_get_str(imp_ord_or_name, 0x100) # imp_ord_or_name = imp_ord_or_name[:imp_ord_or_name.find('\x00')] - #/!\ can have multiple dst ad + #[!] can have multiple dst ad if not imp_ord_or_name in self.lib_imp2dstad[libad]: self.lib_imp2dstad[libad][imp_ord_or_name] = set() if dst_ad is not None: diff --git a/miasm/loader/minidump.py b/miasm/loader/minidump.py index fbb7bde5b..c16473b48 100644 --- a/miasm/loader/minidump.py +++ b/miasm/loader/minidump.py @@ -388,7 +388,7 @@ def check_context(ctx): ("MxCsr", "u32"), # Segment & processor - # /!\ activation depends on multiple flags + # [!] activation depends on multiple flags ("SegCs", "u16", is_activated("CONTEXT_CONTROL")), ("SegDs", "u16", is_activated("CONTEXT_SEGMENTS")), ("SegEs", "u16", is_activated("CONTEXT_SEGMENTS")), @@ -406,7 +406,7 @@ def check_context(ctx): ("Dr7", "u64", is_activated("CONTEXT_DEBUG_REGISTERS")), # Integer registers - # /!\ activation depends on multiple flags + # [!] activation depends on multiple flags ("Rax", "u64", is_activated("CONTEXT_INTEGER")), ("Rcx", "u64", is_activated("CONTEXT_INTEGER")), ("Rdx", "u64", is_activated("CONTEXT_INTEGER")), From ad82438c448597b0c6470eef697a2c1b5f721ef1 Mon Sep 17 00:00:00 2001 From: Duncan Ogilvie Date: Sat, 6 Jan 2024 18:08:46 +0100 Subject: [PATCH 4/5] Use regex literals for re.* functions --- miasm/__init__.py | 6 +++--- miasm/arch/x86/arch.py | 2 +- miasm/core/cpu.py | 4 ++-- miasm/core/graph.py | 2 +- miasm/core/sembuilder.py | 4 ++-- miasm/core/utils.py | 2 +- miasm/ir/ir.py | 2 +- miasm/os_dep/linux/environment.py | 4 ++-- test/analysis/depgraph.py | 4 ++-- test/arch/mep/asm/ut_helpers_asm.py | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/miasm/__init__.py b/miasm/__init__.py index 417a62685..309a1ae7e 100644 --- a/miasm/__init__.py +++ b/miasm/__init__.py @@ -40,13 +40,13 @@ def _version_from_git_describe(): if process.returncode == 0: tag = out.decode().strip() - match = re.match('^v?(.+?)-(\\d+)-g[a-f0-9]+$', tag) + match = re.match(r'^v?(.+?)-(\d+)-g[a-f0-9]+$', tag) if match: # remove the 'v' prefix and add a '.devN' suffix return '%s.dev%s' % (match.group(1), match.group(2)) else: # just remove the 'v' prefix - return re.sub('^v', '', tag) + return re.sub(r'^v', '', tag) else: raise subprocess.CalledProcessError(process.returncode, err) @@ -71,7 +71,7 @@ def _version(): # See 'man gitattributes' for more details. git_archive_id = '$Format:%h %d$' sha1 = git_archive_id.strip().split()[0] - match = re.search('tag:(\\S+)', git_archive_id) + match = re.search(r'tag:(\S+)', git_archive_id) if match: return "git-archive.dev" + match.group(1) elif sha1: diff --git a/miasm/arch/x86/arch.py b/miasm/arch/x86/arch.py index dabd0c82b..c5ff9b631 100644 --- a/miasm/arch/x86/arch.py +++ b/miasm/arch/x86/arch.py @@ -428,7 +428,7 @@ def offsize(p): def get_prefix(s): - g = re.search('(\S+)(\s+)', s) + g = re.search(r'(\S+)(\s+)', s) if not g: return None, s prefix, b = g.groups() diff --git a/miasm/core/cpu.py b/miasm/core/cpu.py index 7a1cacff2..dae93bf9c 100644 --- a/miasm/core/cpu.py +++ b/miasm/core/cpu.py @@ -408,7 +408,7 @@ def cb_op_mul(tokens): def isbin(s): - return re.match('[0-1]+$', s) + return re.match(r'[0-1]+$', s) def int2bin(i, l): @@ -1301,7 +1301,7 @@ def dis(cls, bs_o, mode_o = None, offset=0): @classmethod def fromstring(cls, text, loc_db, mode = None): global total_scans - name = re.search('(\S+)', text).groups() + name = re.search(r'(\S+)', text).groups() if not name: raise ValueError('cannot find name', text) name = name[0] diff --git a/miasm/core/graph.py b/miasm/core/graph.py index 0dfd7e6ac..e680894cd 100644 --- a/miasm/core/graph.py +++ b/miasm/core/graph.py @@ -20,7 +20,7 @@ def __init__(self): # N -> Nodes N2 with a edge (N2 -> N) self._nodes_pred = {} - self.escape_chars = re.compile('[' + re.escape('{}') + '&|<>' + ']') + self.escape_chars = re.compile(r'[\{\}&|<>]') def __repr__(self): diff --git a/miasm/core/sembuilder.py b/miasm/core/sembuilder.py index 244706562..9843ee6a8 100644 --- a/miasm/core/sembuilder.py +++ b/miasm/core/sembuilder.py @@ -22,8 +22,8 @@ class MiasmTransformer(ast.NodeTransformer): """ # Parsers - parse_integer = re.compile("^i([0-9]+)$") - parse_mem = re.compile("^mem([0-9]+)$") + parse_integer = re.compile(r"^i([0-9]+)$") + parse_mem = re.compile(r"^mem([0-9]+)$") # Visitors def visit_Call(self, node): diff --git a/miasm/core/utils.py b/miasm/core/utils.py index 41bf78c1d..eb170576d 100644 --- a/miasm/core/utils.py +++ b/miasm/core/utils.py @@ -26,7 +26,7 @@ COLOR_MNEMO = "blue1" -ESCAPE_CHARS = re.compile('[' + re.escape('{}') + '&|<>' + ']') +ESCAPE_CHARS = re.compile(r'[\{\}&|<>]') def set_html_text_color(text, color): return '%s' % (color, text) diff --git a/miasm/ir/ir.py b/miasm/ir/ir.py index e9b86899f..d26c5d1d5 100644 --- a/miasm/ir/ir.py +++ b/miasm/ir/ir.py @@ -48,7 +48,7 @@ def _expr_loc_to_symb(expr, loc_db): return m2_expr.ExprId(name, expr.size) -ESCAPE_CHARS = re.compile('[' + re.escape('{}') + '&|<>' + ']') +ESCAPE_CHARS = re.compile(r'[\{\}&|<>]') class TranslatorHtml(Translator): __LANG__ = "custom_expr_color" diff --git a/miasm/os_dep/linux/environment.py b/miasm/os_dep/linux/environment.py index 808fc847d..3ba4382f3 100644 --- a/miasm/os_dep/linux/environment.py +++ b/miasm/os_dep/linux/environment.py @@ -13,7 +13,7 @@ from miasm.jitter.csts import PAGE_READ, PAGE_WRITE -REGEXP_T = type(re.compile('')) +REGEXP_T = type(re.compile(r'')) StatInfo = namedtuple("StatInfo", [ "st_dev", "st_ino", "st_nlink", "st_mode", "st_uid", "st_gid", "st_rdev", @@ -262,7 +262,7 @@ def _convert_re(expr): expr.flags, exc_info=True, ) - return re.compile('$X') + return re.compile(r'$X') return expr # Remove '../', etc. diff --git a/test/analysis/depgraph.py b/test/analysis/depgraph.py index 57a73a5f6..9760e7179 100644 --- a/test/analysis/depgraph.py +++ b/test/analysis/depgraph.py @@ -108,7 +108,7 @@ def get_out_regs(self, _): def bloc2graph(irgraph, label=False, lines=True): """Render dot graph of @blocks""" - escape_chars = re.compile('[' + re.escape('{}') + ']') + escape_chars = re.compile(r'[\{\}]') label_attr = 'colspan="2" align="center" bgcolor="grey"' edge_attr = 'label = "%s" color="%s" style="bold"' td_attr = 'align="left"' @@ -179,7 +179,7 @@ def bloc2graph(irgraph, label=False, lines=True): def dg2graph(graph, label=False, lines=True): """Render dot graph of @blocks""" - escape_chars = re.compile('[' + re.escape('{}') + ']') + escape_chars = re.compile(r'[\{\}]') label_attr = 'colspan="2" align="center" bgcolor="grey"' edge_attr = 'label = "%s" color="%s" style="bold"' td_attr = 'align="left"' diff --git a/test/arch/mep/asm/ut_helpers_asm.py b/test/arch/mep/asm/ut_helpers_asm.py index 9f6dc5c20..2ebd0622a 100644 --- a/test/arch/mep/asm/ut_helpers_asm.py +++ b/test/arch/mep/asm/ut_helpers_asm.py @@ -27,7 +27,7 @@ def check_instruction(mn_str, mn_hex, multi=None, offset=0): """Try to disassemble and assemble this instruction""" # Rename objdump registers names - mn_str = re.sub("\$([0-9]+)", lambda m: "R"+m.group(1), mn_str) + mn_str = re.sub(r"\$([0-9]+)", lambda m: "R"+m.group(1), mn_str) mn_str = mn_str.replace("$", "") # Disassemble From 2bf07ad1eb835e12b4b3dbaf42f71adf888a8cc1 Mon Sep 17 00:00:00 2001 From: Duncan Ogilvie Date: Sat, 6 Jan 2024 18:34:48 +0100 Subject: [PATCH 5/5] Replace deprecated use of 'imp' module --- miasm/ir/translators/z3_ir.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/miasm/ir/translators/z3_ir.py b/miasm/ir/translators/z3_ir.py index 4b674c4e3..c72ff36f1 100644 --- a/miasm/ir/translators/z3_ir.py +++ b/miasm/ir/translators/z3_ir.py @@ -1,10 +1,11 @@ from builtins import map from builtins import range -import imp +import importlib.util import logging # Raise an ImportError if z3 is not available WITHOUT actually importing it -imp.find_module("z3") +if importlib.util.find_spec("z3") is None: + raise ImportError("No module named 'z3'") from miasm.ir.translators.translator import Translator