Skip to content

Commit

Permalink
Merge pull request #80 from tubular/fix/sparkly/disable-spark3-kafka-…
Browse files Browse the repository at this point in the history
…read

FIX disable kafka reading for spark 3
  • Loading branch information
srstrickland committed Nov 3, 2020
2 parents c61591c + 5c4c626 commit 194a2f4
Showing 1 changed file with 5 additions and 2 deletions.
7 changes: 5 additions & 2 deletions sparkly/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@
except ImportError:
from urlparse import urlparse, parse_qsl

from pyspark.streaming.kafka import KafkaUtils, OffsetRange

from sparkly.utils import parse_schema


Expand Down Expand Up @@ -247,6 +245,11 @@ def kafka(self,
Raises:
InvalidArgumentError
"""
try:
from pyspark.streaming.kafka import KafkaUtils, OffsetRange
except ImportError:
raise NotImplementedError('Reading from kafka not supported')

if not key_deserializer or not value_deserializer or not schema:
raise InvalidArgumentError('You should specify all of parameters:'
'`key_deserializer`, `value_deserializer` and `schema`')
Expand Down

0 comments on commit 194a2f4

Please sign in to comment.