Merge pull request #10 from nnadeau/fix/rate-limit

engnadeau · Feb 2, 2022 · a0323fe · a0323fe
2 parents 49f8e71 + f7ce820
commit a0323fe
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 43 deletions.
diff --git a/.github/workflows/twitter-friends.yml b/.github/workflows/twitter-friends.yml
@@ -29,4 +29,4 @@ jobs:
           SOCIAL_MEDIA_CRON_TWITTER__ACCESS_TOKEN: ${{ secrets.TWITTER_ACCESS_TOKEN }}
           SOCIAL_MEDIA_CRON_TWITTER__ACCESS_TOKEN_SECRET: ${{ secrets.TWITTER_ACCESS_TOKEN_SECRET }}
         run: |
-          pipenv run python src/twitter.py prune_friends
+          pipenv run python src/twitter.py prune_friends --nowait_on_rate_limit
diff --git a/.github/workflows/twitter-tweets.yml b/.github/workflows/twitter-tweets.yml
@@ -29,4 +29,4 @@ jobs:
           SOCIAL_MEDIA_CRON_TWITTER__ACCESS_TOKEN: ${{ secrets.TWITTER_ACCESS_TOKEN }}
           SOCIAL_MEDIA_CRON_TWITTER__ACCESS_TOKEN_SECRET: ${{ secrets.TWITTER_ACCESS_TOKEN_SECRET }}
         run: |
-          pipenv run python src/twitter.py prune_tweets
+          pipenv run python src/twitter.py prune_tweets --nowait_on_rate_limit
diff --git a/src/twitter.py b/src/twitter.py
@@ -14,7 +14,7 @@
 LOGGER = utils.get_logger("twitter")
 
 
-def _authenticate():
+def _authenticate(wait_on_rate_limit: bool = True):
     """Create authenticated API object."""
     LOGGER.info("Authenticating...")
     auth = tweepy.OAuthHandler(
@@ -23,7 +23,7 @@ def _authenticate():
     auth.set_access_token(
         settings.twitter.access_token, settings.twitter.access_token_secret
     )
-    api = tweepy.API(auth, wait_on_rate_limit=True)
+    api = tweepy.API(auth, wait_on_rate_limit=wait_on_rate_limit)
 
     try:
         user = api.verify_credentials()
@@ -112,6 +112,7 @@ def prune_tweets(
     days: int = settings.twitter.tweet_prune_days,
     delete_liked: bool = settings.twitter.is_delete_liked,
     dry_run: bool = False,
+    wait_on_rate_limit: bool = True,
 ):
     """Delete old tweets.
 
@@ -130,32 +131,44 @@ def prune_tweets(
     limit = TODAY - datetime.timedelta(days=days)
     LOGGER.info(f"Tweets older than {days} days ({limit}) will be deleted")
 
-    api = _authenticate()
+    api = _authenticate(wait_on_rate_limit=wait_on_rate_limit)
     LOGGER.info(f"Authenticated as {api.verify_credentials().screen_name}")
 
     LOGGER.info("Fetching tweets...")
-    for tweet in tweepy.Cursor(method=api.user_timeline, count=100).items():
-        # check if outside time limit and if self-liked
-        if tweet.created_at.date() < limit and (not tweet.favorited or delete_liked):
-            LOGGER.info(
-                f"Pruning: {tweet.id_str} | {tweet.created_at.date()} | {tweet.text}"
-            )
-
-            # skip if dry run
-            if dry_run:
-                continue
-
-            # delete tweet
-            try:
-                api.destroy_status(tweet.id_str)
-            except tweepy.TweepyException as e:
-                LOGGER.error(e)
+    try:
+        for tweet in tweepy.Cursor(method=api.user_timeline, count=100).items():
+            # check if outside time limit and if self-liked
+            if tweet.created_at.date() < limit and (
+                not tweet.favorited or delete_liked
+            ):
+                LOGGER.info(
+                    (
+                        "Pruning: "
+                        f"{tweet.id_str} |"
+                        f"{tweet.created_at.date()} |"
+                        f"{tweet.text}"
+                    )
+                )
+
+                # skip if dry run
+                if dry_run:
+                    continue
+
+                # delete tweet
+                try:
+                    api.destroy_status(tweet.id_str)
+                except tweepy.TweepyException as e:
+                    LOGGER.error(e)
+    except tweepy.errors.TooManyRequests as e:
+        LOGGER.error(e)
 
     LOGGER.info("Pruning complete")
 
 
 def prune_friends(
-    days: int = settings.twitter.friend_prune_days, dry_run: bool = False
+    days: int = settings.twitter.friend_prune_days,
+    dry_run: bool = False,
+    wait_on_rate_limit: bool = True,
 ):
     """Delete old friends.
 
@@ -174,35 +187,40 @@ def prune_friends(
     LOGGER.info(f"Friends inactive since {days} days will be unfriended")
 
     # auth
-    api = _authenticate()
+    api = _authenticate(wait_on_rate_limit=wait_on_rate_limit)
 
     # prune
     screen_name = api.verify_credentials().screen_name
     friends_count = api.verify_credentials().friends_count
     LOGGER.info(f"{screen_name} has {friends_count} friends")
 
     LOGGER.info("Fetching friends...")
-    for friend in tweepy.Cursor(method=api.get_friends, count=200).items():
-        try:
-            days_since_last_tweet = (TODAY - friend.status.created_at.date()).days
-            is_stale_friend = days_since_last_tweet > days
-        except AttributeError:
-            LOGGER.info(f"{friend.screen_name} has never tweeted")
-            days_since_last_tweet = None
-            is_stale_friend = True
-
-        if is_stale_friend:
-            LOGGER.info(f"Pruning: {friend.screen_name} | {days_since_last_tweet} days")
-
-            # skip if dry run
-            if dry_run:
-                continue
-
-            # unfriend
+    try:
+        for friend in tweepy.Cursor(method=api.get_friends, count=200).items():
             try:
-                api.destroy_friendship(screen_name=friend.screen_name)
-            except tweepy.TweepyException as e:
-                LOGGER.error(e)
+                days_since_last_tweet = (TODAY - friend.status.created_at.date()).days
+                is_stale_friend = days_since_last_tweet > days
+            except AttributeError:
+                LOGGER.info(f"{friend.screen_name} has never tweeted")
+                days_since_last_tweet = None
+                is_stale_friend = True
+
+            if is_stale_friend:
+                LOGGER.info(
+                    f"Pruning: {friend.screen_name} | {days_since_last_tweet} days"
+                )
+
+                # skip if dry run
+                if dry_run:
+                    continue
+
+                # unfriend
+                try:
+                    api.destroy_friendship(screen_name=friend.screen_name)
+                except tweepy.TweepyException as e:
+                    LOGGER.error(e)
+    except tweepy.errors.TooManyRequests as e:
+        LOGGER.error(e)
 
     LOGGER.info("Pruning complete")