Skip to content
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
45 lines (32 sloc) 1.23 KB
"""Batch processing with MapReduce, implemented with Dramatiq."""
import itertools
import dramatiq
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
from dramatiq.brokers.redis import RedisBroker
import requests
import spacy
# Dramatiq Setup
result_backend = RedisBackend(host="redis", port=6379)
redis_broker = RedisBroker(host="redis", port=6379)
# Using Spacy for entity extraction.
nlp = spacy.load("en_core_web_sm")
def extract_ents(paragraph: str):
"""Extract entities from the paragraph."""
doc = nlp(paragraph)
ents = []
for ent in doc.ents:
if ent.label_ == "PERSON":
return ents
if __name__ == "__main__":
# Moby Dick - Split into paragraphs.
data = requests.get("").content.decode('utf-8')
paragraphs = data.split("\r\n\r\n")
print("Staring Jobs")
group =[extract_ents.message(x) for x in paragraphs]).run()
results = itertools.chain(*[x for x in group.get_results(block=True, timeout=100_000)])
You can’t perform that action at this time.