From 9fd73fce99b802cf6ac937494f1df3a0b82fc784 Mon Sep 17 00:00:00 2001 From: Krystian Owoc Date: Mon, 4 Mar 2024 14:47:26 +0100 Subject: [PATCH 1/4] Added patreon post comments --- gallery_dl/extractor/patreon.py | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index 62d11f23e6..ee771905d1 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -133,6 +133,7 @@ def _process(self, post, included): """Process and extend a 'post' object""" attr = post["attributes"] attr["id"] = text.parse_int(post["id"]) + self._process_post_comments(attr) if attr.get("current_user_can_view", True): @@ -156,6 +157,34 @@ def _process(self, post, included): return attr + def _process_post_comments(self, post): + headers = { + "Content-Type": "application/vnd.api+json", + } + + post["comments"] = [] + + comments_url = text.ensure_http_scheme(PatreonExtractor._build_comments_url(post["id"])) + while comments_url: + comments = self.request(comments_url, headers=headers).json() + post["comments"].extend(comments["data"]) + comments_url = comments["links"].get("next") + + for comment in post['comments']: + self._process_comment_replies(comment) + + def _process_comment_replies(self, comment): + headers = { + "Content-Type": "application/vnd.api+json", + } + + replies_url = text.ensure_http_scheme(PatreonExtractor._build_comment_replies_url(comment["id"])) + comments = self.request(replies_url, headers=headers).json() + comment["replies"] = comments["data"] + + for reply in comment['replies']: + self._process_comment_replies(reply) + @staticmethod def _transform(included): """Transform 'included' into an easier to handle format""" @@ -243,6 +272,14 @@ def _build_url(endpoint, query): "&json-api-version=1.0" ) + @staticmethod + def _build_comments_url(post_id): + return "https://www.patreon.com/api/posts/" + str(post_id) + "/comments" + + @staticmethod + def _build_comment_replies_url(comment_id): + return "https://www.patreon.com/api/comments/" + str(comment_id) + "/replies" + def _build_file_generators(self, filetypes): if filetypes is None: return (self._images, self._image_large, From 287b7171450f65227a627cb2ca864b2965407c7f Mon Sep 17 00:00:00 2001 From: Krystian Owoc Date: Mon, 4 Mar 2024 15:09:22 +0100 Subject: [PATCH 2/4] Fixed lines width --- gallery_dl/extractor/patreon.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index ee771905d1..96e3862b18 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -164,7 +164,8 @@ def _process_post_comments(self, post): post["comments"] = [] - comments_url = text.ensure_http_scheme(PatreonExtractor._build_comments_url(post["id"])) + comments_url = text.ensure_http_scheme( + PatreonExtractor._build_comments_url(post["id"])) while comments_url: comments = self.request(comments_url, headers=headers).json() post["comments"].extend(comments["data"]) @@ -178,7 +179,8 @@ def _process_comment_replies(self, comment): "Content-Type": "application/vnd.api+json", } - replies_url = text.ensure_http_scheme(PatreonExtractor._build_comment_replies_url(comment["id"])) + replies_url = text.ensure_http_scheme( + PatreonExtractor._build_comment_replies_url(comment["id"])) comments = self.request(replies_url, headers=headers).json() comment["replies"] = comments["data"] @@ -274,11 +276,17 @@ def _build_url(endpoint, query): @staticmethod def _build_comments_url(post_id): - return "https://www.patreon.com/api/posts/" + str(post_id) + "/comments" + return ( + "https://www.patreon.com/api/posts/" + str(post_id) + + "/comments" + ) @staticmethod def _build_comment_replies_url(comment_id): - return "https://www.patreon.com/api/comments/" + str(comment_id) + "/replies" + return ( + "https://www.patreon.com/api/comments/" + str(comment_id) + + "/replies" + ) def _build_file_generators(self, filetypes): if filetypes is None: From 85901656cd2e1b79d00b1a9b03c0a3059d8143b0 Mon Sep 17 00:00:00 2001 From: Krystian Owoc Date: Tue, 5 Mar 2024 18:00:45 +0100 Subject: [PATCH 3/4] Fixed 403 error when trying to download comments from locked post --- gallery_dl/extractor/patreon.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index 96e3862b18..1199a02253 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -133,10 +133,12 @@ def _process(self, post, included): """Process and extend a 'post' object""" attr = post["attributes"] attr["id"] = text.parse_int(post["id"]) - self._process_post_comments(attr) + if attr.get("current_user_can_view", True): + self._process_post_comments(attr) + relationships = post["relationships"] attr["images"] = self._files(post, included, "images") attr["attachments"] = self._files(post, included, "attachments") From a84014f151b9b3e7c6f704282f7d3cef291dd284 Mon Sep 17 00:00:00 2001 From: Krystian Owoc Date: Tue, 5 Mar 2024 18:03:11 +0100 Subject: [PATCH 4/4] Removed blank line --- gallery_dl/extractor/patreon.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index 1199a02253..7b8c8625df 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -134,7 +134,6 @@ def _process(self, post, included): attr = post["attributes"] attr["id"] = text.parse_int(post["id"]) - if attr.get("current_user_can_view", True): self._process_post_comments(attr)