speed up extension loading (#267)

* try to avoid slow pkg_resources calls

* use importlib-resources instead of pkg_resources

* fix collection of packages to log

mapchete/_registered.py

@@ -0,0 +1,9 @@
+try:
+    from importlib import metadata
+except ImportError:
+    # <PY38 use backport
+    import importlib_metadata as metadata
+
+commands = metadata.entry_points().get("mapchete.cli.commands", ())
+drivers = metadata.entry_points().get("mapchete.formats.drivers", ())
+processes = metadata.entry_points().get("mapchete.processes", ())

mapchete/cli/default/create.py

@@ -1,30 +1,30 @@
 """Create dummy Mapchete and python process files."""
 
 import click
+from importlib_resources import files
 import os
 from string import Template
 from shutil import copyfile
 from oyaml import dump
-import pkg_resources
 
 from mapchete.cli import utils
 
 FORMAT_MANDATORY = {
     "GTiff": {
         "bands": None,
         "dtype": None
-        },
+    },
     "PNG": {
         "bands": None,
         "dtype": None
-        },
+    },
     "PNG_hillshade": {
         "bands": 4,
         "dtype": "uint8"
-        },
+    },
     "GeoJSON": {
         "schema": {}
-        },
+    },
     "PostGIS": {
         "schema": {},
         "db_params": {
@@ -34,9 +34,9 @@
             "user": None,
             "password": None,
             "table": None
-            }
         }
     }
+}
 
 
 @click.command(help="Create a new process.")
@@ -62,15 +62,14 @@ def create(
     out_path = out_path if out_path else os.path.join(os.getcwd(), "output")
 
     # copy file template to target directory
-    process_template = pkg_resources.resource_filename(
-        "mapchete.static", "process_template.py"
-    )
+    # Reads contents with UTF-8 encoding and returns str.
+    process_template = str(files("mapchete.static").joinpath("process_template.py"))
     process_file = os.path.join(os.getcwd(), process_file)
     copyfile(process_template, process_file)
 
     # modify and copy mapchete file template to target directory
-    mapchete_template = pkg_resources.resource_filename(
-        "mapchete.static", "mapchete_template.mapchete"
+    mapchete_template = str(
+        files("mapchete.static").joinpath("mapchete_template.mapchete")
     )
 
     output_options = dict(

mapchete/cli/main.py

@@ -1,16 +1,13 @@
-"""
-Mapchete command line tool with subcommands.
-"""
-
-from pkg_resources import iter_entry_points
+"""Mapchete command line tool with subcommands."""
 
 import click
 from click_plugins import with_plugins
 
 from mapchete import __version__
+from mapchete._registered import commands
 
 
-@with_plugins(iter_entry_points('mapchete.cli.commands'))
+@with_plugins(commands)
 @click.version_option(version=__version__, message='%(version)s')
 @click.group()
 def main():

mapchete/formats/__init__.py

@@ -7,18 +7,16 @@
 import fiona
 import logging
 import os
-import pkg_resources
 from pprint import pformat
 import rasterio
 from rasterio.crs import CRS
 import warnings
 
 from mapchete.errors import MapcheteConfigError, MapcheteDriverError
 from mapchete.io import read_json, write_json, path_exists
+from mapchete._registered import drivers
 from mapchete.tile import BufferedTilePyramid
 
-DRIVERS_ENTRY_POINT = "mapchete.formats.drivers"
-
 
 logger = logging.getLogger(__name__)
 
@@ -33,7 +31,7 @@ def available_output_formats():
         all available output formats
     """
     output_formats = {}
-    for v in pkg_resources.iter_entry_points(DRIVERS_ENTRY_POINT):
+    for v in drivers:
         driver_ = v.load()
         if hasattr(driver_, "METADATA") and (driver_.METADATA["mode"] in ["w", "rw"]):
             output_formats[driver_.METADATA["driver_name"]] = driver_.METADATA
@@ -50,7 +48,7 @@ def available_input_formats():
         all available input formats
     """
     input_formats = {}
-    for v in pkg_resources.iter_entry_points(DRIVERS_ENTRY_POINT):
+    for v in drivers:
         logger.debug("driver found: %s", v)
         driver_ = v.load()
         if hasattr(driver_, "METADATA") and (driver_.METADATA["mode"] in ["r", "rw"]):
@@ -70,7 +68,7 @@ def load_output_reader(output_params):
     if not isinstance(output_params, dict):
         raise TypeError("output_params must be a dictionary")
     driver_name = output_params["format"]
-    for v in pkg_resources.iter_entry_points(DRIVERS_ENTRY_POINT):
+    for v in drivers:
         _driver = v.load()
         if all(
             [hasattr(_driver, attr) for attr in ["OutputDataReader", "METADATA"]]
@@ -93,7 +91,7 @@ def load_output_writer(output_params, readonly=False):
     if not isinstance(output_params, dict):
         raise TypeError("output_params must be a dictionary")
     driver_name = output_params["format"]
-    for v in pkg_resources.iter_entry_points(DRIVERS_ENTRY_POINT):
+    for v in drivers:
         _driver = v.load()
         if all(
             [hasattr(_driver, attr) for attr in ["OutputDataWriter", "METADATA"]]
@@ -127,7 +125,7 @@ def load_input_reader(input_params, readonly=False):
             driver_name = "TileDirectory"
     else:
         raise MapcheteDriverError("invalid input parameters %s" % input_params)
-    for v in pkg_resources.iter_entry_points(DRIVERS_ENTRY_POINT):
+    for v in drivers:
         driver_ = v.load()
         if hasattr(driver_, "METADATA") and (
             driver_.METADATA["driver_name"] == driver_name

mapchete/log.py

@@ -7,16 +7,11 @@
 """
 from itertools import chain
 import logging
-import pkg_resources
 import warnings
 
-all_mapchete_packages = set(
-    v.module_name.split(".")[0]
-    for v in chain(
-        pkg_resources.iter_entry_points("mapchete.formats.drivers"),
-        pkg_resources.iter_entry_points("mapchete.processes")
-    )
-)
+from mapchete._registered import drivers, processes
+
+all_mapchete_packages = set(v.value.split(".")[0] for v in chain(drivers, processes))
 
 key_value_replace_patterns = {
     "AWS_ACCESS_KEY_ID": "***",

mapchete/processes/__init__.py

@@ -1,5 +1,6 @@
 import logging
-import pkg_resources
+
+from mapchete._registered import processes
 
 logger = logging.getLogger(__name__)
 
@@ -18,8 +19,6 @@ def registered_processes(process_name=None):
     module
     """
     def _import():
-        # get all registered processes by name
-        processes = list(pkg_resources.iter_entry_points("mapchete.processes"))
         # try to load processes
         for v in processes:
             logger.debug("try to load %s", v)

requirements.txt

@@ -4,6 +4,8 @@ click>=7.1.1
 click-plugins
 click-spinner
 fiona>=1.8b1
+importlib-metadata
+importlib-resources
 numpy>=1.16
 oyaml
 retry

setup.py

@@ -91,6 +91,4 @@ def _parse_requirements(file):
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
     ],
-    setup_requires=["pytest-runner"],
-    tests_require=["pytest", "pytest-flask", "rio-cogeo"]
 )