Avoid using __file__ in places that matter

Assuming __file__ exists breaks packaging netaddr with PyOxidirzer[1].

Initially I wanted to use pkgutil.get_data()[2] as it's been in the Python
standard library since Python 2.6 but it always reads and returns the
whole resource and I decided I don't like reading whole out.txt and
iab.txt in OUI and IAB constructors just to read few small bits of data.
importlib.resources provides an API[3] that should in the usual cases
cases[4] avoid loading whole resources into memory. Maybe IAB and OUI
constructors shouldn't read files and the API needs to be rethought but
that's an issue for another day.

Granted, this is a tradeoff, as we have a dependency now which means
slighly more network traffic and slightly more complexity, but all
things considered I think that's better than the alternative.

(Ironically this introduces a new piece of code using __file__ but this
should be benign as it's not in the code that'll be present in
a PyOxidizer-produced binary. It's necessary to have that setup.py hack
as netaddr can't be imported without importlib_resources or
importlib.resources present now so it can't be unconditionally imported
from setup.py where importlib_resources may not be installed yet).

Fixes GH-188.

[1] indygreg/PyOxidizer#69
[2] https://docs.python.org/2/library/pkgutil.html#pkgutil.get_data
[3] https://docs.python.org/3.9/library/importlib.html#importlib.resources.open_binary
[4] https://gitlab.com/python-devs/importlib_resources/-/blob/2707fb7384e76cda715de14bea5956339969950f/importlib_resources/_py3.py#L24

.github/workflows/ci.yml

@@ -18,6 +18,7 @@ jobs:
     - name: Install dependencies
       run: |
         pip install --upgrade -r requirements.txt
+        pip install .
     - name: Run tests
       run: |
         make test_with_junitxml

netaddr/compat.py

@@ -113,3 +113,7 @@ def _iter_next(x):
     raise RuntimeError(
         'this module only supports Python 2.4.x or higher (including 3.x)!')
 
+try:
+    from importlib import resources as _importlib_resources
+except ImportError:
+    import importlib_resources as _importlib_resources

netaddr/eui/__init__.py

@@ -13,7 +13,7 @@
 from netaddr.strategy.eui48 import mac_eui48
 from netaddr.strategy.eui64 import eui64_base
 from netaddr.ip import IPAddress
-from netaddr.compat import _is_int, _is_str
+from netaddr.compat import _importlib_resources, _is_int, _is_str
 
 
 class BaseIdentifier(object):
@@ -91,7 +91,7 @@ def __init__(self, oui):
 
         #   Discover offsets.
         if self._value in ieee.OUI_INDEX:
-            fh = open(ieee.OUI_REGISTRY_PATH, 'rb')
+            fh = _importlib_resources.open_binary(__package__, 'oui.txt')
             for (offset, size) in ieee.OUI_INDEX[self._value]:
                 fh.seek(offset)
                 data = fh.read(size).decode('UTF-8')
@@ -256,7 +256,7 @@ def __init__(self, iab, strict=False):
 
         #   Discover offsets.
         if self._value in ieee.IAB_INDEX:
-            fh = open(ieee.IAB_REGISTRY_PATH, 'rb')
+            fh = _importlib_resources.open_binary(__package__, 'iab.txt')
             (offset, size) = ieee.IAB_INDEX[self._value][0]
             self.record['offset'] = offset
             self.record['size'] = size

netaddr/eui/ieee.py

@@ -35,24 +35,13 @@
 import os.path as _path
 import csv as _csv
 
-from netaddr.compat import _bytes_type
+from netaddr.compat import _bytes_type, _importlib_resources
 from netaddr.core import Subscriber, Publisher
 
 
-#: Path to local copy of IEEE OUI Registry data file.
-OUI_REGISTRY_PATH = _path.join(_path.dirname(__file__), 'oui.txt')
-#: Path to netaddr OUI index file.
-OUI_INDEX_PATH = _path.join(_path.dirname(__file__), 'oui.idx')
-
 #: OUI index lookup dictionary.
 OUI_INDEX = {}
 
-#: Path to local copy of IEEE IAB Registry data file.
-IAB_REGISTRY_PATH = _path.join(_path.dirname(__file__), 'iab.txt')
-
-#: Path to netaddr IAB index file.
-IAB_INDEX_PATH = _path.join(_path.dirname(__file__), 'iab.idx')
-
 #: IAB index lookup dictionary.
 IAB_INDEX = {}
 
@@ -258,22 +247,29 @@ def parse(self):
         self.notify(record)
 
 
-def create_index_from_registry(registry_path, index_path, parser):
+def create_index_from_registry(registry_fh, index_path, parser):
     """Generate an index files from the IEEE registry file."""
-    oui_parser = parser(registry_path)
+    oui_parser = parser(registry_fh)
     oui_parser.attach(FileIndexer(index_path))
     oui_parser.parse()
 
 
 def create_indices():
     """Create indices for OUI and IAB file based lookups"""
-    create_index_from_registry(OUI_REGISTRY_PATH, OUI_INDEX_PATH, OUIIndexParser)
-    create_index_from_registry(IAB_REGISTRY_PATH, IAB_INDEX_PATH, IABIndexParser)
-
-
-def load_index(index, index_path):
+    create_index_from_registry(
+        _path.join(_path.dirname(__file__), 'oui.txt'),
+        _path.join(_path.dirname(__file__), 'oui.idx'),
+        OUIIndexParser,
+    )
+    create_index_from_registry(
+        _path.join(_path.dirname(__file__), 'iab.txt'),
+        _path.join(_path.dirname(__file__), 'iab.idx'),
+        IABIndexParser,
+    )
+
+
+def load_index(index, fp):
     """Load index from file into index data structure."""
-    fp = open(index_path, 'rb')
     try:
         for row in _csv.reader([x.decode('UTF-8') for x in fp]):
             (key, offset, size) = [int(_) for _ in row]
@@ -285,8 +281,8 @@ def load_index(index, index_path):
 
 def load_indices():
     """Load OUI and IAB lookup indices into memory"""
-    load_index(OUI_INDEX, OUI_INDEX_PATH)
-    load_index(IAB_INDEX, IAB_INDEX_PATH)
+    load_index(OUI_INDEX, _importlib_resources.open_binary(__package__, 'oui.idx'))
+    load_index(IAB_INDEX, _importlib_resources.open_binary(__package__, 'iab.idx'))
 
 
 if __name__ == '__main__':

netaddr/ip/iana.py

@@ -29,13 +29,12 @@
     - IEEE Protocols Information Home Page - http://www.iana.org/protocols/
 """
 
-import os.path as _path
 import sys as _sys
 from xml.sax import make_parser, handler
 
 from netaddr.core import Publisher, Subscriber
 from netaddr.ip import IPAddress, IPNetwork, IPRange, cidr_abbrev_to_verbose
-from netaddr.compat import _dict_items, _callable
+from netaddr.compat import _dict_items, _callable, _importlib_resources
 
 
 
@@ -362,21 +361,21 @@ def load_info():
     Parse and load internal IANA data lookups with the latest information from
     data files.
     """
-    PATH = _path.dirname(__file__)
-
-    ipv4 = IPv4Parser(open(_path.join(PATH, 'ipv4-address-space.xml')))
+    ipv4 = IPv4Parser(_importlib_resources.open_binary(__package__, 'ipv4-address-space.xml'))
     ipv4.attach(DictUpdater(IANA_INFO['IPv4'], 'IPv4', 'prefix'))
     ipv4.parse()
 
-    ipv6 = IPv6Parser(open(_path.join(PATH, 'ipv6-address-space.xml')))
+    ipv6 = IPv6Parser(_importlib_resources.open_binary(__package__, 'ipv6-address-space.xml'))
     ipv6.attach(DictUpdater(IANA_INFO['IPv6'], 'IPv6', 'prefix'))
     ipv6.parse()
 
-    ipv6ua = IPv6UnicastParser(open(_path.join(PATH, 'ipv6-unicast-address-assignments.xml')))
+    ipv6ua = IPv6UnicastParser(
+        _importlib_resources.open_binary(__package__, 'ipv6-unicast-address-assignments.xml'),
+    )
     ipv6ua.attach(DictUpdater(IANA_INFO['IPv6_unicast'], 'IPv6_unicast', 'prefix'))
     ipv6ua.parse()
 
-    mcast = MulticastParser(open(_path.join(PATH, 'multicast-addresses.xml')))
+    mcast = MulticastParser(_importlib_resources.open_binary(__package__, 'multicast-addresses.xml'))
     mcast.attach(DictUpdater(IANA_INFO['multicast'], 'multicast', 'address'))
     mcast.parse()
 

netaddr/tests/eui/test_ieee_parsers.py

@@ -1,18 +1,17 @@
-import os
+import contextlib
 import sys
 
 import pytest
 
+from netaddr.compat import _importlib_resources
 from netaddr.eui.ieee import OUIIndexParser, IABIndexParser, FileIndexer
 
 
-SAMPLE_DIR = os.path.dirname(__file__)
-
 @pytest.mark.skipif(sys.version_info > (3,), reason="requires python 2.x")
 def test_oui_parser_py2():
     from cStringIO import StringIO
     outfile = StringIO()
-    with open(os.path.join(SAMPLE_DIR, 'sample_oui.txt'), 'rb') as infile:
+    with contextlib.closing(_importlib_resources.open_binary(__package__, 'sample_oui.txt')) as infile:
         iab_parser = OUIIndexParser(infile)
         iab_parser.attach(FileIndexer(outfile))
         iab_parser.parse()
@@ -23,7 +22,7 @@ def test_oui_parser_py2():
 def test_iab_parser_py2():
     from cStringIO import StringIO
     outfile = StringIO()
-    with open(os.path.join(SAMPLE_DIR, 'sample_iab.txt'), 'rb') as infile:
+    with contextlib.closing(_importlib_resources.open_binary(__package__, 'sample_iab.txt')) as infile:
         iab_parser = IABIndexParser(infile)
         iab_parser.attach(FileIndexer(outfile))
         iab_parser.parse()
@@ -34,7 +33,7 @@ def test_iab_parser_py2():
 def test_oui_parser_py3():
     from io import StringIO
     outfile = StringIO()
-    with open(os.path.join(SAMPLE_DIR, 'sample_oui.txt'), 'rb') as infile:
+    with contextlib.closing(_importlib_resources.open_binary(__package__, 'sample_oui.txt')) as infile:
         iab_parser = OUIIndexParser(infile)
         iab_parser.attach(FileIndexer(outfile))
         iab_parser.parse()
@@ -45,7 +44,7 @@ def test_oui_parser_py3():
 def test_iab_parser_py3():
     from io import StringIO
     outfile = StringIO()
-    with open(os.path.join(SAMPLE_DIR, 'sample_iab.txt'), 'rb') as infile:
+    with contextlib.closing(_importlib_resources.open_binary(__package__, 'sample_iab.txt')) as infile:
         iab_parser = IABIndexParser(infile)
         iab_parser.attach(FileIndexer(outfile))
         iab_parser.parse()

setup.py

@@ -13,8 +13,6 @@
 if os.path.exists('MANIFEST'):
     os.remove('MANIFEST')
 
-import netaddr
-
 keywords = [
     'Networking', 'Systems Administration', 'IANA', 'IEEE', 'CIDR', 'IP',
     'IPv4', 'IPv6', 'CIDR', 'EUI', 'MAC', 'MAC-48', 'EUI-48', 'EUI-64'
@@ -171,7 +169,16 @@ def main():
         platforms=platforms,
         entry_points={'console_scripts': ['netaddr = netaddr.cli:main']},
         url='https://github.com/drkjam/netaddr/',
-        version=netaddr.__version__,
+        version=(
+            [
+                ln for ln in open(os.path.join(os.path.dirname(__file__), 'netaddr', '__init__.py'))
+                if '__version__' in ln
+            ][0]
+            .split('=')[-1]
+            .strip()
+            .strip('\'"')
+        ),
+        install_requires=['importlib-resources;python_version<"3.7"'],
     )
 
     setup(**setup_options)