Skip to content

Commit

Permalink
Remove "spacy-transformers" package from `importlib.metadata.entry_po…
Browse files Browse the repository at this point in the history
…ints()`

to prevent spaCy to load the "transformers" package when loading CPU
model in calibre's Python interpreter.
  • Loading branch information
xxyzz committed Dec 5, 2022
1 parent 3ad53d5 commit 11dd530
Showing 1 changed file with 19 additions and 0 deletions.
19 changes: 19 additions & 0 deletions parse_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
from .x_ray import X_Ray
from .x_ray_share import NER_LABELS, CustomX, get_custom_x_path, load_custom_x_desc
except ImportError:
isfrozen = False
from database import (
create_lang_layer,
create_x_ray_db,
Expand Down Expand Up @@ -575,6 +576,11 @@ def find_named_entity(


def load_spacy(model: str, book_path: str) -> Any:
if not model.endswith("_trf") and isfrozen:
import importlib.metadata

importlib.metadata.distributions = hide_spacy_transformers

import spacy

excluded_components = [
Expand Down Expand Up @@ -604,3 +610,16 @@ def load_spacy(model: str, book_path: str) -> Any:
patterns.append({"label": label, "pattern": alias, "id": name})
ruler.add_patterns(patterns)
return nlp


def hide_spacy_transformers(**kwargs):
"""
Remove spacy-transfomers package from the results of importlib.metadata.entry_points().
spaCy loads the spacy-transfomers packge even when loading a CPU model, which loads
code from the transformers library and causes error in calibre's Python interpreter.
Traceback log can be found in GitHub issue #91.
"""
from importlib.metadata import Distribution

pkgs = Distribution.discover(**kwargs)
return [pkg for pkg in pkgs if pkg.name != "spacy-transformers"]

0 comments on commit 11dd530

Please sign in to comment.