[*chan] Add text-posts opt (generic text-tweets)

gallery_dl/extractor/2chan.py

@@ -44,17 +44,19 @@ class _2chanThreadExtractor(Extractor):
     def __init__(self, match):
         Extractor.__init__(self, match)
         self.server, self.board, self.thread = match.groups()
+        self.text_posts = self.config("text-posts", False)
 
     def items(self):
         url = "https://{}.2chan.net/{}/res/{}.htm".format(
             self.server, self.board, self.thread)
         page = self.request(url).text
         data = self.metadata(page)
-        yield Message.Directory, data
         for post in self.posts(page):
+            post.update(data)
+            if "filename" in post or self.text_posts:
+                yield Message.Directory, post
             if "filename" not in post:
                 continue
-            post.update(data)
             url = self.url_fmt.format_map(post)
             yield Message.Url, url, post
 

gallery_dl/extractor/420chan.py

@@ -27,6 +27,7 @@ class _420chanThreadExtractor(Extractor):
     def __init__(self, match):
         Extractor.__init__(self, match)
         self.board, self.thread = match.groups()
+        self.text_posts = self.config("text-posts", False)
 
     def items(self):
         url = "https://api.420chan.org/{}/res/{}.json".format(
@@ -39,10 +40,11 @@ def items(self):
             "title" : posts[0].get("sub") or posts[0]["com"][:50],
         }
 
-        yield Message.Directory, data
         for post in posts:
+            post.update(data)
+            if "filename" in post or self.text_posts:
+                yield Message.Directory, post
             if "filename" in post:
-                post.update(data)
                 post["extension"] = post["ext"][1:]
                 url = "https://boards.420chan.org/{}/src/{}{}".format(
                     post["board"], post["filename"], post["ext"])

gallery_dl/extractor/4chan.py

@@ -36,6 +36,7 @@ class _4chanThreadExtractor(Extractor):
     def __init__(self, match):
         Extractor.__init__(self, match)
         self.board, self.thread = match.groups()
+        self.text_posts = self.config("text-posts", False)
 
     def items(self):
         url = "https://a.4cdn.org/{}/thread/{}.json".format(
@@ -49,10 +50,11 @@ def items(self):
             "title" : text.unescape(title)[:50],
         }
 
-        yield Message.Directory, data
         for post in posts:
+            post.update(data)
+            if "filename" in post or self.text_posts:
+                yield Message.Directory, post
             if "filename" in post:
-                post.update(data)
                 post["extension"] = post["ext"][1:]
                 post["filename"] = text.unescape(post["filename"])
                 url = "https://i.4cdn.org/{}/{}{}".format(

gallery_dl/extractor/8chan.py

@@ -103,6 +103,7 @@ class _8chanThreadExtractor(_8chanExtractor):
     def __init__(self, match):
         _8chanExtractor.__init__(self, match)
         _, self.board, self.thread = match.groups()
+        self.text_posts = self.config("text-posts", False)
 
     def items(self):
         # fetch thread data
@@ -120,12 +121,11 @@ def items(self):
 
         # download files
         posts = thread.pop("posts", ())
-        yield Message.Directory, thread
         for post in itertools.chain((thread,), posts):
             files = post.pop("files", ())
-            if not files:
-                continue
             thread.update(post)
+            if files or self.text_posts:
+                yield Message.Directory, post
             for num, file in enumerate(files):
                 file.update(thread)
                 file["num"] = num

gallery_dl/extractor/lynxchan.py

@@ -62,24 +62,25 @@ def __init__(self, match):
         index = match.lastindex
         self.board = match.group(index-1)
         self.thread = match.group(index)
+        self.text_posts = self.config("text-posts", False)
 
     def items(self):
         url = "{}/{}/res/{}.json".format(self.root, self.board, self.thread)
         thread = self.request(url).json()
         thread["postId"] = thread["threadId"]
         posts = thread.pop("posts", ())
 
-        yield Message.Directory, thread
         for post in itertools.chain((thread,), posts):
             files = post.pop("files", ())
-            if files:
-                thread.update(post)
-                for num, file in enumerate(files):
-                    file.update(thread)
-                    file["num"] = num
-                    url = self.root + file["path"]
-                    text.nameext_from_url(file["originalName"], file)
-                    yield Message.Url, url, file
+            thread.update(post)
+            if files or self.text_posts:
+                yield Message.Directory, thread
+            for num, file in enumerate(files):
+                file.update(thread)
+                file["num"] = num
+                url = self.root + file["path"]
+                text.nameext_from_url(file["originalName"], file)
+                yield Message.Url, url, file
 
 
 class LynxchanBoardExtractor(LynxchanExtractor):

gallery_dl/extractor/twitter.py

@@ -31,7 +31,9 @@ class TwitterExtractor(Extractor):
     def __init__(self, match):
         Extractor.__init__(self, match)
         self.user = match.group(1)
-        self.textonly = self.config("text-tweets", False)
+        self.textonly = self.config("text-posts", False) or self.config(
+            "text-tweets", False
+        )
         self.retweets = self.config("retweets", False)
         self.replies = self.config("replies", True)
         self.twitpic = self.config("twitpic", False)

gallery_dl/extractor/vichan.py

@@ -73,6 +73,7 @@ def __init__(self, match):
         index = match.lastindex
         self.board = match.group(index-1)
         self.thread = match.group(index)
+        self.text_posts = self.config("text-posts", False)
 
     def items(self):
         url = "{}/{}/res/{}.json".format(self.root, self.board, self.thread)
@@ -87,8 +88,10 @@ def items(self):
             "num"   : 0,
         }
 
-        yield Message.Directory, data
         for post in posts:
+            post.update(data)
+            if "filename" in post or self.text_posts:
+                yield Message.Directory, post
             if "filename" in post:
                 yield process(post, data)
                 if "extra_files" in post:
@@ -97,15 +100,13 @@ def items(self):
                         yield process(post, filedata)
 
     def _process(self, post, data):
-        post.update(data)
         post["extension"] = post["ext"][1:]
         post["url"] = "{}/{}/src/{}{}".format(
             self.root, post["board"], post["tim"], post["ext"])
         return Message.Url, post["url"], post
 
     @staticmethod
     def _process_8kun(post, data):
-        post.update(data)
         post["extension"] = post["ext"][1:]
 
         tim = post["tim"]