Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for an initial before_attribute in stream_generator #1983

Merged
merged 9 commits into from Oct 8, 2023
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions AUTHORS.rst
Expand Up @@ -84,4 +84,5 @@ Source Contributors
- Josh Kim `@jsk56143 <https://github.com/jsk56143>`_
- Rolf Campbell `@endlisnis <https://github.com/endlisnis>`_
- zacc `@zacc <https://github.com/zacc>`_
- c0d3rman `@c0d3rman <https://github.com/c0d3rman>`
- Add "Name <email (optional)> and github profile link" above this line.
12 changes: 10 additions & 2 deletions praw/models/util.py
Expand Up @@ -35,7 +35,12 @@ def permissions_string(


@_deprecate_args(
"function", "pause_after", "skip_existing", "attribute_name", "exclude_before"
"function",
"pause_after",
"skip_existing",
"attribute_name",
"exclude_before",
"start_after",
)
def stream_generator(
function: Callable,
Expand All @@ -44,6 +49,7 @@ def stream_generator(
exclude_before: bool = False,
pause_after: int | None = None,
skip_existing: bool = False,
start_after: str | None = None,
c0d3rman marked this conversation as resolved.
Show resolved Hide resolved
**function_kwargs: Any,
) -> Generator[Any, None, None]:
"""Yield new items from ``function`` as they become available.
Expand All @@ -62,6 +68,8 @@ def stream_generator(
:param skip_existing: When ``True``, this does not yield any results from the first
request thereby skipping any items that existed in the stream prior to starting
the stream (default: ``False``).
:param start_after: The initial string value to use for ``before`` in ``params``.
c0d3rman marked this conversation as resolved.
Show resolved Hide resolved
The stream will start from the item following this one (default: ``None``).

Additional keyword arguments will be passed to ``function``.

Expand Down Expand Up @@ -122,7 +130,7 @@ def stream_generator(
print(comment)

"""
before_attribute = None
before_attribute = start_after
exponential_counter = ExponentialCounter(max_counter=16)
seen_attributes = BoundedSet(301)
without_before_counter = 0
Expand Down

Large diffs are not rendered by default.

10 changes: 10 additions & 0 deletions tests/integration/models/reddit/test_subreddit.py
Expand Up @@ -954,6 +954,16 @@ def test_comments__with_skip_existing(self, reddit):
# that there are at least 400 comments in the stream.
assert count < 400

def test_comments__with_start_after(self, reddit):
subreddit = reddit.subreddit("kakapo")
c0d3rman marked this conversation as resolved.
Show resolved Hide resolved
initial_stream = subreddit.stream.comments()
first_ten = [next(initial_stream) for _ in range(10)]
generator = subreddit.stream.comments(start_after=first_ten[4].fullname)
for i in range(5):
comment = next(generator)
assert isinstance(comment, Comment)
assert comment.fullname == first_ten[i + 5].fullname

def test_submissions(self, reddit):
subreddit = reddit.subreddit("all")
generator = subreddit.stream.submissions()
Expand Down
30 changes: 30 additions & 0 deletions tests/unit/models/test_util.py
Expand Up @@ -124,3 +124,33 @@ def generate(limit, **kwargs):
thing = next(stream)
assert thing not in seen
seen.add(thing)

def test_stream_start_after(
self,
):
Thing = namedtuple("Thing", ["fullname"])
initial_things = [Thing(n) for n in reversed(range(100))]
counter = 99

def generate(limit, params=None, **kwargs):
nonlocal counter
counter += 1
sliced_things = initial_things
if params:
sliced_things = initial_things[
: next(
i
for i, thing in enumerate(initial_things)
if thing.fullname == params["before"]
)
]
if counter % 2 == 0:
return sliced_things
return [Thing(counter)] + sliced_things[:-1]

stream = stream_generator(generate, start_after=49)
expected_fullname = 50
for _ in range(50):
thing = next(stream)
assert thing.fullname == expected_fullname, thing
expected_fullname += 1