From f76ca6d9b4d3d130ceb7fb08b47c5cec369e6ede Mon Sep 17 00:00:00 2001 From: gpotter2 Date: Wed, 2 Jun 2021 02:07:07 +0200 Subject: [PATCH 1/4] Test for imports --- test/import_tester | 3 --- test/imports.uts | 63 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) delete mode 100644 test/import_tester create mode 100644 test/imports.uts diff --git a/test/import_tester b/test/import_tester deleted file mode 100644 index eebaba60135..00000000000 --- a/test/import_tester +++ /dev/null @@ -1,3 +0,0 @@ -#! /bin/bash -cd "$(dirname $0)/.." -find scapy -name '*.py' | sed -e 's#/#.#g' -e 's/\(\.__init__\)\?\.py$//' | while read a; do echo "######### $a"; python -c "import $a"; done diff --git a/test/imports.uts b/test/imports.uts new file mode 100644 index 00000000000..aa24f6fc555 --- /dev/null +++ b/test/imports.uts @@ -0,0 +1,63 @@ +% Import tests + ++ Import tests +~ python3_only imports + += Prepare importing all scapy files + +import glob +import subprocess + +# DEV: to add your file to this list, make sure you have +# a GREAT reason. +EXCEPTIONS = [ + "scapy.main", + "scapy.__main__", + "scapy.contrib.cansocket_python_can", +] +EXCEPTION_PACKAGES = [ + "arch", + "tools", + "module" +] + +ALL_FILES = [ + "scapy." + re.match(r".*scapy\/(.*)\.py$", x).group(1).replace("/", ".") + for x in glob.iglob(scapy_path('/scapy/**/*.py'), recursive=True) +] +ALL_FILES = [ + x for x in ALL_FILES if + x not in EXCEPTIONS and + x.split(".")[1] not in EXCEPTION_PACKAGES +] + + +def import_all(FILES): + for file in FILES: + proc = subprocess.Popen( + [sys.executable, "-c", + "import %s" % file], + stderr=subprocess.PIPE, + encoding="utf8") + errs = "" + try: + outs, errs = proc.communicate(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + errs = "Timed out !" + if proc.returncode != 0: + print("Importing the file '%s' failed !" % file) + print(errs) + raise ImportError + += Try importing all core separately + +import_all(x for x in ALL_FILES if "layers" not in x and "contrib" not in x) + += Try importing all layers separately + +import_all(x for x in ALL_FILES if "layers" in x) + += Try importing all contribs separately + +import_all(x for x in ALL_FILES if "contrib" in x) From 5d8e7ab5dbf59c945b04a7f0c6eea50e36274d3e Mon Sep 17 00:00:00 2001 From: gpotter2 Date: Sun, 6 Jun 2021 00:58:52 +0200 Subject: [PATCH 2/4] Fix BPF import loop --- scapy/arch/bpf/supersocket.py | 2 +- test/imports.uts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scapy/arch/bpf/supersocket.py b/scapy/arch/bpf/supersocket.py index 8587e6fa6b5..7f7fc406be8 100644 --- a/scapy/arch/bpf/supersocket.py +++ b/scapy/arch/bpf/supersocket.py @@ -24,7 +24,6 @@ from scapy.interfaces import network_name from scapy.supersocket import SuperSocket from scapy.compat import raw -from scapy.layers.l2 import Loopback if FREEBSD: @@ -375,6 +374,7 @@ def recv(self, x=BPF_BUFFER_LENGTH): def send(self, pkt): """Send a packet""" + from scapy.layers.l2 import Loopback # Use the routing table to find the output interface iff = pkt.route()[0] diff --git a/test/imports.uts b/test/imports.uts index aa24f6fc555..8619bef687a 100644 --- a/test/imports.uts +++ b/test/imports.uts @@ -41,7 +41,7 @@ def import_all(FILES): encoding="utf8") errs = "" try: - outs, errs = proc.communicate(timeout=3) + _, errs = proc.communicate(timeout=10) except subprocess.TimeoutExpired: proc.kill() errs = "Timed out !" From 04768880baa7bb7d53872579379fcecf1f352b27 Mon Sep 17 00:00:00 2001 From: gpotter2 Date: Sat, 3 Jul 2021 00:20:02 +0200 Subject: [PATCH 3/4] Greatly speedup imports --- test/imports.uts | 54 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/test/imports.uts b/test/imports.uts index 8619bef687a..99a5970ec99 100644 --- a/test/imports.uts +++ b/test/imports.uts @@ -31,24 +31,46 @@ ALL_FILES = [ x.split(".")[1] not in EXCEPTION_PACKAGES ] +import importlib +from multiprocessing import Pool + +process_file_code = """# This was automatically generated +import subprocess, sys + +def process_file(file): + proc = subprocess.Popen( + [sys.executable, "-c", + "import %s" % file], + stderr=subprocess.PIPE, + encoding="utf8") + errs = "" + try: + _, errs = proc.communicate(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + errs = "Timed out !" + if proc.returncode != 0: + return "Importing the file '%s' failed !\\n%s" % (file, errs) + return None +""" + +tmp = get_temp_file(autoext=".py", keep=True) +print(tmp) +with open(tmp, "w") as fd: + fd.write(process_file_code) + +fld, file = os.path.split(tmp) +sys.path.append(fld) +pkg = importlib.import_module(os.path.splitext(file)[0]) def import_all(FILES): - for file in FILES: - proc = subprocess.Popen( - [sys.executable, "-c", - "import %s" % file], - stderr=subprocess.PIPE, - encoding="utf8") - errs = "" - try: - _, errs = proc.communicate(timeout=10) - except subprocess.TimeoutExpired: - proc.kill() - errs = "Timed out !" - if proc.returncode != 0: - print("Importing the file '%s' failed !" % file) - print(errs) - raise ImportError + with Pool(processes=8) as pool: + for err in pool.imap_unordered(pkg.process_file, FILES, 4): + if err: + print(err) + pool.terminate() + raise ImportError + = Try importing all core separately From f4ac0ba8b72af39b78bc5b979ec3dfff291d6613 Mon Sep 17 00:00:00 2001 From: gpotter2 Date: Tue, 13 Jul 2021 19:00:55 +0200 Subject: [PATCH 4/4] Disable import on some modules --- test/imports.uts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/imports.uts b/test/imports.uts index 99a5970ec99..8c7e02a89b7 100644 --- a/test/imports.uts +++ b/test/imports.uts @@ -11,14 +11,18 @@ import subprocess # DEV: to add your file to this list, make sure you have # a GREAT reason. EXCEPTIONS = [ - "scapy.main", "scapy.__main__", - "scapy.contrib.cansocket_python_can", + "scapy.contrib.automotive*", + "scapy.contrib.cansocket*", + "scapy.contrib.isotp*", + "scapy.contrib.scada*", + "scapy.main", ] EXCEPTION_PACKAGES = [ "arch", + "libs", + "modules", "tools", - "module" ] ALL_FILES = [ @@ -27,7 +31,7 @@ ALL_FILES = [ ] ALL_FILES = [ x for x in ALL_FILES if - x not in EXCEPTIONS and + not any(x == y if y[-1] != "*" else x.startswith(y[:-1]) for y in EXCEPTIONS) and x.split(".")[1] not in EXCEPTION_PACKAGES ]