add tqdm progress bars

docs/gettingstarted/installation.md

@@ -34,12 +34,15 @@ First you need to install ```pleroma-bot``` on your system. There are multiple m
 
     Either way, here's a list of the dependencies in case you need them:
 
-    | Name                     | Git repo                                                | Docs                                                         |
-    |--------------------------|---------------------------------------------------------|--------------------------------------------------------------|
-    | python-oauthlib          | [GitHub](https://github.com/oauthlib/oauthlib)          | [Documentation](https://oauthlib.readthedocs.io/en/latest/)  |
-    | python-pyaml             | [GitHub](https://github.com/yaml/pyyaml)                | [Documentation](https://pyyaml.org/wiki/PyYAMLDocumentation) |
-    | python-requests          | [GitHub](https://github.com/psf/requests)               | [Documentation](https://requests.readthedocs.io/)            |
-    | python-requests-oauthlib | [GitHub](https://github.com/requests/requests-oauthlib) | [Documentation](https://requests-oauthlib.readthedocs.org)   |
+    | Name                     | Git repo                                                      | Docs                                                                    |
+    |--------------------------|---------------------------------------------------------------|-------------------------------------------------------------------------|
+    | python-oauthlib          | [GitHub](https://github.com/oauthlib/oauthlib)                | [Documentation](https://oauthlib.readthedocs.io/en/latest/)             |
+    | python-pyaml             | [GitHub](https://github.com/yaml/pyyaml)                      | [Documentation](https://pyyaml.org/wiki/PyYAMLDocumentation)            |
+    | python-requests          | [GitHub](https://github.com/psf/requests)                     | [Documentation](https://requests.readthedocs.io/)                       |
+    | python-requests-oauthlib | [GitHub](https://github.com/requests/requests-oauthlib)       | [Documentation](https://requests-oauthlib.readthedocs.org)              |
+    | feedparser               | [GitHub](https://github.com/kurtmckee/feedparser)             | [Documentation](https://feedparser.readthedocs.io/en/latest/)           |
+    | beautifulsoup4           | [Webpage](https://www.crummy.com/software/BeautifulSoup/bs4/) | [Documentation](https://www.crummy.com/software/BeautifulSoup/bs4/doc/) |
+    | tqdm                     | [GitHub](https://github.com/tqdm/tqdm)                        | [Documentation](https://tqdm.github.io/)                                |
 
 ## Test the installation
 

pleroma_bot/__init__.py

@@ -2,6 +2,7 @@
 import sys
 import locale
 import logging
+from tqdm._utils import _term_move_up
 
 __version__ = "1.1.0"
 
@@ -16,28 +17,29 @@ class CustomFormatter(logging.Formatter):
     yellow = "\x1b[33;21m"
     red = "\x1b[31;21m"
     bold_red = "\x1b[31;1m"
-    reset = "\x1b[0m"
+    reset = "\x1b[0m\x1b[80D\x1b[1A\x1b[K"
     format_r = "%(asctime)s - %(name)s - %(levelname)s - %(message)s "
     format_l = (
         "%(asctime)s - %(name)s - %(levelname)s - %(message)s "
         "(%(filename)s:%(lineno)d) "
     )
-
+    border = "=" * 50
+    clear_border = _term_move_up() + "\r" + " " * len(border) + "\r"
     if sys.platform != "win32" and sys.stdout.encoding.casefold() == 'utf-8':
         FORMATS = {
-            logging.DEBUG: grey + format_r + reset,
-            logging.INFO: grey + "ℹ " + format_r + reset,
-            logging.WARNING: yellow + "⚠ " + format_l + reset,
-            logging.ERROR: red + "✖ " + format_l + reset,
-            logging.CRITICAL: bold_red + format_l + reset,
+            logging.DEBUG: clear_border + grey + format_r + reset,
+            logging.INFO: clear_border + grey + "ℹ " + format_r + reset,
+            logging.WARNING: clear_border + yellow + "⚠ " + format_l + reset,
+            logging.ERROR: clear_border + red + "✖ " + format_l + reset,
+            logging.CRITICAL: clear_border + bold_red + format_l + reset,
         }
     else:  # pragma: win32 cover
         FORMATS = {
-            logging.DEBUG: format_r,
-            logging.INFO: "¡ " + format_r,
-            logging.WARNING: "!!" + format_l,
-            logging.ERROR: "× " + format_l,
-            logging.CRITICAL: "× " + format_l,
+            logging.DEBUG: '\n' + format_r,
+            logging.INFO: '\n' + "¡ " + format_r,
+            logging.WARNING: '\n' + "!!" + format_l,
+            logging.ERROR: '\n' + "× " + format_l,
+            logging.CRITICAL: '\n' + "× " + format_l,
         }
 
     def format(self, record):

pleroma_bot/_pleroma.py

@@ -91,7 +91,7 @@ def post_pleroma(
         )
         if not response.ok:
             response.raise_for_status()
-        logger.info(_("Reblog in Pleroma:\t{}").format(str(response)))
+        logger.debug(_("Reblog in Pleroma:\t{}").format(str(response)))
         return post_id
 
     media_ids = []
@@ -208,7 +208,7 @@ def post_pleroma(
         )
         if not response.ok:
             response.raise_for_status()
-        logger.info(_("Post in Pleroma:\t{}").format(str(response)))
+        logger.debug(_("Post in Pleroma:\t{}").format(str(response)))
         post_id = json.loads(response.text)["id"]
         self.posts_ids[self.pleroma_base_url].update({tweet_id: post_id})
     return post_id

pleroma_bot/_processing.py

@@ -7,6 +7,7 @@
 import mimetypes
 from datetime import datetime
 
+from tqdm import tqdm
 
 # Try to import libmagic
 # if it fails just use mimetypes
@@ -78,7 +79,12 @@ def process_tweets(self, tweets_to_post):
                 tweets_to_post["data"].remove(tweet)
                 pass
     all_media = []
-    for tweet in tweets_to_post["data"]:
+    if int(self.threads) == 1:
+        desc = _("Processing tweets... ")
+        pbar = tqdm(total=len(tweets_to_post["data"]), desc=desc)
+
+    for idx, tweet in enumerate(tweets_to_post["data"]):
+        # logger.info(_("Processing: {}/{}").format(idx+1, total_tweets))
         self.posts_ids[self.pleroma_base_url].update({tweet["id"]: ''})
         tweet["reply_id"] = None
         tweet["retweet_id"] = None
@@ -203,6 +209,8 @@ def process_tweets(self, tweets_to_post):
                 ).format(self.max_post_length)
             )
             tweet["text"] = f"{tweet['text'][:self.max_post_length]}"
+        if int(self.threads) == 1:
+            pbar.update(1)
     tweets_to_post["media_processed"] = all_media
     return tweets_to_post
 

pleroma_bot/_twitter.py

@@ -1,11 +1,13 @@
 import time
 import requests
 
+from tqdm import tqdm
 from datetime import datetime
 
 from . import logger
 from pleroma_bot.i18n import _
-from pleroma_bot._utils import spinner
+
+# from pleroma_bot._utils import spinner
 
 
 def twitter_api_request(method, url,
@@ -149,7 +151,8 @@ def _get_tweets(
         version: str,
         tweet_id=None,
         start_time=None,
-        t_user=None):
+        t_user=None,
+        pbar=None):
     """Gathers last 'max_tweets' tweets from the user and returns them
     as an dict
     :param version: Twitter API version to use to retrieve the tweets
@@ -198,7 +201,7 @@ def _get_tweets(
             return tweets
     elif version == "v2":
         tweets_v2 = self._get_tweets_v2(
-            tweet_id=tweet_id, start_time=start_time, t_user=t_user
+            tweet_id=tweet_id, start_time=start_time, t_user=t_user, pbar=pbar
         )
         return tweets_v2
     else:
@@ -213,7 +216,8 @@ def _get_tweets_v2(
         previous_token=None,
         count=0,
         tweets_v2=None,
-        t_user=None
+        t_user=None,
+        pbar=None
 ):
     if not (3200 >= self.max_tweets >= 10):
         global _
@@ -353,7 +357,8 @@ def _get_tweets_v2(
         tweets_v2["meta"] = next_tweets["meta"]
     else:
         tweets_v2 = response.json()
-
+    if pbar:
+        pbar.update(response.json()["meta"]["result_count"])
     try:
         next_token = response.json()["meta"]["next_token"]
         count += response.json()["meta"]["result_count"]
@@ -364,16 +369,18 @@ def _get_tweets_v2(
                 next_token=next_token,
                 previous_token=previous_token,
                 count=count,
-                t_user=t_user
+                t_user=t_user,
+                pbar=pbar
             )
     except KeyError:
         pass
 
     return tweets_v2
 
 
-@spinner(_("Gathering tweets... "))
+# @spinner(_("Gathering tweets... "))
 def get_tweets(self, start_time):
+    from .i18n import _
     t_utweets = {}
     self.result_count = 0
     tweets_merged = {
@@ -382,11 +389,15 @@ def get_tweets(self, start_time):
     }
     try:
         for t_user in self.twitter_username:
+            desc = _("Gathering tweets... ")
+            pbar = tqdm(desc=desc, position=0, total=10000, bar_format='{desc}{n_fmt}')
             t_utweets[t_user] = self._get_tweets(
                 "v2",
                 start_time=start_time,
-                t_user=t_user
+                t_user=t_user,
+                pbar=pbar
             )
+            pbar.close()
             self.result_count += t_utweets[t_user]["meta"]["result_count"]
             tweets_merged["meta"] = {}
 

pleroma_bot/_utils.py

@@ -13,11 +13,12 @@
 import functools
 import mimetypes
 
+from tqdm import tqdm
 from typing import cast
 from errno import ENOENT
 from itertools import cycle
 from bs4 import BeautifulSoup
-from multiprocessing import Queue
+from multiprocessing import Queue, Pool
 from json.decoder import JSONDecodeError
 from datetime import datetime, timedelta
 from itertools import tee, islice, chain
@@ -102,15 +103,24 @@ def chunkify(lst, n):
 def process_parallel(tweets, user, threads):
     dt = tweets["data"]
     chunks = chunkify(dt, threads)
-    mp = Multiprocessor()
+    # mp = Multiprocessor()
+    tweets_chunked = []
     for idx in range(threads):
-        tweets_chunked = {
+        tweets_chunked.append({
             "data": chunks[idx],
             "includes": tweets["includes"],
             "meta": tweets["meta"]
-        }
-        mp.run(user.process_tweets, tweets_chunked)
-    ret = mp.wait()  # get all results
+        })
+    with Pool(threads) as p:
+        ret = []
+        desc = _("Processing tweets... ")
+        with tqdm(total=len(dt), desc=desc) as pbar:
+            for idx, res in enumerate(
+                    p.imap_unordered(user.process_tweets, tweets_chunked)
+            ):
+                pbar.update(len(chunks[idx]))
+                ret.append(res)
+
     tweets_merged = {
         "data": [],
         "includes": tweets["includes"],
@@ -143,7 +153,7 @@ def run(self, func, *args, **kwargs):
         self.processes.append(p)
         p.start()
 
-    @spinner(_("Processing tweets... "), 1.2)
+    # @spinner(_("Processing tweets... "), 1.2)
     def wait(self):
         rets = []
         for p in self.processes:
@@ -775,6 +785,8 @@ def get_date_last_post(self):
         date = self.get_date_last_pleroma_post()
     elif instance == "misskey":
         date = self.get_date_last_misskey_post()
+    elif instance == "cohost":
+        date = self.get_date_last_cohost_post()
     return date
 
 

pleroma_bot/cli.py

@@ -38,6 +38,7 @@
 import argparse
 import multiprocessing as mp
 
+from tqdm import tqdm
 from random import shuffle
 from requests_oauthlib import OAuth1
 from requests.structures import CaseInsensitiveDict
@@ -69,6 +70,10 @@ class User(object):
     from ._misskey import update_misskey
     from ._misskey import get_date_last_misskey_post
 
+    from ._cohost import _login_cohost
+    from ._cohost import _get_cohost_profile_info
+    from ._cohost import get_date_last_cohost_post
+
     from ._utils import pin
     from ._utils import post
     from ._utils import unpin
@@ -147,7 +152,8 @@ def __init__(self, user_cfg: dict, cfg: dict, base_path: str, posts_ids):
             "include_quotes": True,
             "website": None,
             "no_profile": False,
-            "rss": None
+            "rss": None,
+            "threads": 1
         }
         # iterate attrs defined in config
         for attribute in default_cfg_attributes:
@@ -225,8 +231,12 @@ def __init__(self, user_cfg: dict, cfg: dict, base_path: str, posts_ids):
         os.makedirs(self.tweets_temp_path, exist_ok=True)
         for t_user in t_users:
             os.makedirs(self.user_path[t_user], exist_ok=True)
-        # Get Fedi instance info
-        self._get_instance_info()
+        if self.pleroma_base_url == "https://cohost.org":
+            self.instance = "cohost"
+            self._get_cohost_profile_info()
+        else:
+            # Get Fedi instance info
+            self._get_instance_info()
         if self.rss:
             self.skip_pin = True
             self.no_profile = True
@@ -578,6 +588,7 @@ def main():
                     else:
                         cores = mp.cpu_count()
                         threads = round(cores / 2 if cores > 4 else 4)
+                    user.threads = threads
                     if user.rss:
                         tweets_to_post = tweets_rss
                     else:
@@ -597,9 +608,11 @@ def main():
                     )
                     tweet_counter = 0
                     posted = {}
+                    desc = _("Posting tweets... ")
+                    pbar = tqdm(total=len(tweets_to_post["data"]), desc=desc)
                     for tweet in tweets_to_post["data"]:
                         tweet_counter += 1
-                        logger.info(
+                        logger.debug(
                             f"({tweet_counter}/{len(tweets_to_post['data'])})"
                         )
                         try:
@@ -626,6 +639,7 @@ def main():
                         posts_ids = user.posts_ids
                         with open(posts_path, "w") as f:
                             f.write(json.dumps(posts_ids, indent=4))
+                        pbar.update(1)
                         time.sleep(user.delay_post)
                 if not user.skip_pin:
                     user.check_pinned(posted)

requirements.txt

@@ -2,4 +2,5 @@ requests>=2.24.0
 PyYAML>=5.3.1
 requests-oauthlib>=1.3.0
 feedparser>=6.0.10
-beautifulsoup4>=4.9.3
+beautifulsoup4>=4.9.3
+tqdm>=4.64.1

setup.py

@@ -40,7 +40,8 @@
         'PyYAML>=5.3.1',
         'requests-oauthlib>=1.3.0',
         'feedparser>=6.0.10',
-        'beautifulsoup4>=4.9.3'
+        'beautifulsoup4>=4.9.3',
+        'tqdm>=4.64.1',
     ],
     extras_require={
         'lint': [