Defer import of matplotlib until needed (#3579)

* Defer import of matplotlib until needed

matplotlib has a heavy dep chain

Fixes #3578

* relocate

* pyx relo

* update docstrings

* git add file

* fix typo

* fix typo

* naming

* missed one

* adjust logging to try to figure out why it fails on py27 windows

* fix py27 windows

* unguarded raise

* try to make py27 happy

* name collision?

* tweak

* rename to avoid name collision

.config/mypy/mypy_enabled.txt

@@ -30,7 +30,6 @@ scapy/consts.py
 scapy/dadict.py
 scapy/data.py
 scapy/error.py
-scapy/extlib.py
 scapy/fields.py
 scapy/interfaces.py
 scapy/main.py

scapy/layers/inet.py

@@ -27,8 +27,6 @@
     Loopback
 from scapy.compat import raw, chb, orb, bytes_encode, Optional
 from scapy.config import conf
-from scapy.extlib import plt, MATPLOTLIB, MATPLOTLIB_INLINED, \
-    MATPLOTLIB_DEFAULT_PLOT_KARGS
 from scapy.fields import (
     BitEnumField,
     BitField,
@@ -1282,6 +1280,13 @@ def defragment(plist):
 # Add timeskew_graph() method to PacketList
 def _packetlist_timeskew_graph(self, ip, **kargs):
     """Tries to graph the timeskew between the timestamps and real time for a given ip"""  # noqa: E501
+    # Defer imports of matplotlib until its needed
+    # because it has a heavy dep chain
+    from scapy.libs.matplot import (
+        plt,
+        MATPLOTLIB_INLINED,
+        MATPLOTLIB_DEFAULT_PLOT_KARGS
+    )
 
     # Filter TCP segments which source address is 'ip'
     tmp = (self._elt2pkt(x) for x in self.res)
@@ -1534,6 +1539,8 @@ def world_trace(self):
 
         # Check that the geoip2 module can be imported
         # Doc: http://geoip2.readthedocs.io/en/latest/
+        from scapy.libs.matplot import plt, MATPLOTLIB, MATPLOTLIB_INLINED
+
         try:
             # GeoIP2 modules need to be imported as below
             import geoip2.database

scapy/libs/matplot.py

@@ -0,0 +1,43 @@
+# This file is part of Scapy
+# See http://www.secdev.org/projects/scapy for more information
+# Copyright (C) Philippe Biondi <phil@secdev.org>
+# This program is published under a GPLv2 license
+
+"""
+External link to matplotlib
+"""
+
+from scapy.error import log_loading
+
+# Notice: this file must not be called before main.py, if started
+# in interactive mode, because it needs to be called after the
+# logger has been setup, to be able to print the warning messages
+
+__all__ = [
+    "Line2D",
+    "MATPLOTLIB",
+    "MATPLOTLIB_DEFAULT_PLOT_KARGS",
+    "MATPLOTLIB_INLINED",
+    "plt",
+]
+
+# MATPLOTLIB
+
+try:
+    from matplotlib import get_backend as matplotlib_get_backend
+    from matplotlib import pyplot as plt
+    from matplotlib.lines import Line2D
+    MATPLOTLIB = 1
+    if "inline" in matplotlib_get_backend():
+        MATPLOTLIB_INLINED = 1
+    else:
+        MATPLOTLIB_INLINED = 0
+    MATPLOTLIB_DEFAULT_PLOT_KARGS = {"marker": "+"}
+# RuntimeError to catch gtk "Cannot open display" error
+except (ImportError, RuntimeError) as ex:
+    plt = None
+    Line2D = None
+    MATPLOTLIB = 0
+    MATPLOTLIB_INLINED = 0
+    MATPLOTLIB_DEFAULT_PLOT_KARGS = dict()
+    log_loading.info("Can't import matplotlib: %s. Won't be able to plot.", ex)

scapy/extlib.py → scapy/libs/test_pyx.py

@@ -4,7 +4,7 @@
 # This program is published under a GPLv2 license
 
 """
-External link to programs
+External link to pyx
 """
 
 import os
@@ -16,35 +16,9 @@
 # logger has been setup, to be able to print the warning messages
 
 __all__ = [
-    "Line2D",
-    "MATPLOTLIB",
-    "MATPLOTLIB_DEFAULT_PLOT_KARGS",
-    "MATPLOTLIB_INLINED",
     "PYX",
-    "plt",
 ]
 
-# MATPLOTLIB
-
-try:
-    from matplotlib import get_backend as matplotlib_get_backend
-    from matplotlib import pyplot as plt
-    from matplotlib.lines import Line2D
-    MATPLOTLIB = 1
-    if "inline" in matplotlib_get_backend():
-        MATPLOTLIB_INLINED = 1
-    else:
-        MATPLOTLIB_INLINED = 0
-    MATPLOTLIB_DEFAULT_PLOT_KARGS = {"marker": "+"}
-# RuntimeError to catch gtk "Cannot open display" error
-except (ImportError, RuntimeError):
-    plt = None
-    Line2D = None
-    MATPLOTLIB = 0
-    MATPLOTLIB_INLINED = 0
-    MATPLOTLIB_DEFAULT_PLOT_KARGS = dict()
-    log_loading.info("Can't import matplotlib. Won't be able to plot.")
-
 # PYX
 
 

scapy/packet.py

@@ -46,7 +46,7 @@
 from scapy.utils import import_hexcap, tex_escape, colgen, issubtype, \
     pretty_list, EDecimal
 from scapy.error import Scapy_Exception, log_runtime, warning
-from scapy.extlib import PYX
+from scapy.libs.test_pyx import PYX
 import scapy.libs.six as six
 
 # Typing imports

scapy/plist.py

@@ -12,6 +12,7 @@
 from __future__ import print_function
 import os
 from collections import defaultdict
+from typing import TYPE_CHECKING
 
 from scapy.compat import lambda_tuple_converter
 from scapy.config import conf
@@ -24,8 +25,6 @@
 )
 from scapy.utils import do_graph, hexdump, make_table, make_lined_table, \
     make_tex_table, issubtype
-from scapy.extlib import plt, Line2D, \
-    MATPLOTLIB_INLINED, MATPLOTLIB_DEFAULT_PLOT_KARGS
 from functools import reduce
 import scapy.libs.six as six
 
@@ -47,6 +46,8 @@
 )
 from scapy.packet import Packet
 
+if TYPE_CHECKING:
+    from scapy.libs.matplot import Line2D
 
 #############
 #  Results  #
@@ -289,6 +290,13 @@ def plot(self,
 
         lfilter: a truth function that decides whether a packet must be plotted
         """
+        # Defer imports of matplotlib until its needed
+        # because it has a heavy dep chain
+        from scapy.libs.matplot import (
+            plt,
+            MATPLOTLIB_INLINED,
+            MATPLOTLIB_DEFAULT_PLOT_KARGS
+        )
 
         # Python 2 backward compatibility
         f = lambda_tuple_converter(f)
@@ -327,6 +335,13 @@ def diffplot(self,
 
         A list of matplotlib.lines.Line2D is returned.
         """
+        # Defer imports of matplotlib until its needed
+        # because it has a heavy dep chain
+        from scapy.libs.matplot import (
+            plt,
+            MATPLOTLIB_INLINED,
+            MATPLOTLIB_DEFAULT_PLOT_KARGS
+        )
 
         # Get the list of packets
         if lfilter is None:
@@ -360,6 +375,13 @@ def multiplot(self,
 
         A list of matplotlib.lines.Line2D is returned.
         """
+        # Defer imports of matplotlib until its needed
+        # because it has a heavy dep chain
+        from scapy.libs.matplot import (
+            plt,
+            MATPLOTLIB_INLINED,
+            MATPLOTLIB_DEFAULT_PLOT_KARGS
+        )
 
         # Python 2 backward compatibility
         f = lambda_tuple_converter(f)

test/regression.uts

@@ -905,36 +905,37 @@ from mock import patch
 def _r(*args, **kwargs):
     raise OSError
 
-with patch("scapy.extlib.subprocess.check_call", _r):
-    from scapy.extlib import _test_pyx
+with patch("scapy.libs.test_pyx.subprocess.check_call", _r):
+    from scapy.libs.test_pyx import _test_pyx
     assert _test_pyx() == False
 
 = Test matplotlib detection functions
 
 from mock import MagicMock, patch
 
-bck_scapy_ext_lib = sys.modules.get("scapy.extlib", None)
-del(sys.modules["scapy.extlib"])
+bck_scapy_libs_matplot = sys.modules.get("scapy.libs.matplot", None)
+if bck_scapy_libs_matplot:
+    del(sys.modules["scapy.libs.matplot"])
 
 mock_matplotlib = MagicMock()
 mock_matplotlib.get_backend.return_value = "inline"
 mock_matplotlib.pyplot = MagicMock()
 mock_matplotlib.pyplot.plt = None
 with patch.dict("sys.modules", **{ "matplotlib": mock_matplotlib, "matplotlib.lines": mock_matplotlib}):
-    from scapy.extlib import MATPLOTLIB, MATPLOTLIB_INLINED, MATPLOTLIB_DEFAULT_PLOT_KARGS, Line2D
+    from scapy.libs.matplot import MATPLOTLIB, MATPLOTLIB_INLINED, MATPLOTLIB_DEFAULT_PLOT_KARGS, Line2D
     assert MATPLOTLIB == 1
     assert MATPLOTLIB_INLINED == 1
     assert "marker" in MATPLOTLIB_DEFAULT_PLOT_KARGS
 
 mock_matplotlib.get_backend.return_value = "ko"
 with patch.dict("sys.modules", **{ "matplotlib": mock_matplotlib, "matplotlib.lines": mock_matplotlib}):
-    from scapy.extlib import MATPLOTLIB, MATPLOTLIB_INLINED, MATPLOTLIB_DEFAULT_PLOT_KARGS
+    from scapy.libs.matplot import MATPLOTLIB, MATPLOTLIB_INLINED, MATPLOTLIB_DEFAULT_PLOT_KARGS
     assert MATPLOTLIB == 1
     assert MATPLOTLIB_INLINED == 0
     assert "marker" in MATPLOTLIB_DEFAULT_PLOT_KARGS
 
-if bck_scapy_ext_lib:
-    sys.modules["scapy.extlib"] = bck_scapy_ext_lib
+if bck_scapy_libs_matplot:
+    sys.modules["scapy.libs.matplot"] = bck_scapy_libs_matplot
 
 
 ############
@@ -4409,7 +4410,9 @@ assert all(bytes(a[0]) == bytes(b[0]) for a, b in zip(unp, srl))
 = plot()
 
 import mock
-@mock.patch("scapy.plist.plt")
+import scapy.libs.matplot
+
+@mock.patch("scapy.libs.matplot.plt")
 def test_plot(mock_plt):
     def fake_plot(data, **kwargs):
         return data
@@ -4423,7 +4426,9 @@ test_plot()
 = diffplot()
 
 import mock
-@mock.patch("scapy.plist.plt")
+import scapy.libs.matplot
+
+@mock.patch("scapy.libs.matplot.plt")
 def test_diffplot(mock_plt):
     def fake_plot(data, **kwargs):
         return data
@@ -4437,7 +4442,9 @@ test_diffplot()
 = multiplot()
 
 import mock
-@mock.patch("scapy.plist.plt")
+import scapy.libs.matplot
+
+@mock.patch("scapy.libs.matplot.plt")
 def test_multiplot(mock_plt):
     def fake_plot(data, **kwargs):
         return data

test/scapy/layers/inet.uts

@@ -568,8 +568,9 @@ def test_summary():
 test_summary()
 
 import mock
+import scapy.libs.matplot
 
-@mock.patch("scapy.layers.inet.plt")
+@mock.patch("scapy.libs.matplot.plt")
 def test_timeskew_graph(mock_plt):
     def fake_plot(data, **kwargs):
         return data