Merge pull request #213 from GrafeasGroup/honeycomb

Honeycomb support

.github/workflows/automated-tests.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: '3.8'
+          python-version: '3.9'
           architecture: 'x64'
       - run: |
           pip install --upgrade pip
@@ -33,7 +33,7 @@ jobs:
       max-parallel: 4
       matrix:
         python-version:
-          - '3.8'
+          - '3.9'
 
     steps:
       - uses: actions/checkout@v2

pyproject.toml

@@ -15,7 +15,7 @@ exclude = ['test', 'test.*', '*.test.*', '*.test']
 include = ["commands.json", "tor/strings/*.yml"]
 
 [tool.poetry.dependencies]
-python = "^3.6"
+python = "^3.9"
 sh = "^1.12"
 bugsnag = "^3.6"
 requests = "^2.22"
@@ -24,21 +24,40 @@ PyYaml = "^5.1"
 blossom-wrapper = { git = "https://github.com/GrafeasGroup/blossom-wrapper.git", branch = "master" }
 python-dotenv = "^0.14.0"
 praw = "^7.1.0"
+toml = "^0.10.2"
+honeycomb-beeline = "^2.17.0"
 
 [tool.poetry.dev-dependencies]
 better-exceptions = "^0.2.2"
 pytest = "^5.1"
 pytest-cov = "^2.7"
 flake8 = "^3.7.9"
-mypy = "^0.761"
+mypy = ">=0.800"
 black = "^19.10b0"
+types-toml = "^0.1"
+types-PyYAML = "^0.1"
+types-requests = "^0.1"
+types-redis = "^3.5"
 
 [tool.poetry.scripts]
 tor-moderator = "tor.cli.main:main"
 
 [tool.poetry.extras]
 ci = ['pytest', 'pytest-cov']
 
+[[tool.mypy.overrides]]
+module = [
+    "praw",
+    "praw.*",
+    "prawcore",
+    "prawcore.*",
+    "blossom_wrapper",
+    "beeline",
+    "slackclient",
+    "pytest",
+]
+ignore_missing_imports = true
+
 [build-system]
 requires = ["poetry>=0.12"]
 build-backend = "poetry.masonry.api"

scripts/clear_submissions_from_queue.py

@@ -1,13 +1,14 @@
 from datetime import datetime, timedelta
 
 from praw import Reddit
-from praw.exceptions import RedditAPIException
+# from praw.exceptions import RedditAPIException
 
 r = Reddit('bot2')
 
 UNCLAIMED = 'Unclaimed'
 IN_PROGRESS = 'In Progress'
 
+
 def remove_old_crap(posts):
     for item in posts:
         if item.link_flair_text == UNCLAIMED:
@@ -16,22 +17,24 @@ def remove_old_crap(posts):
                 print(f"Removing {item.name}, posted on {str(submission_time)}")
                 item.mod.remove()
 
-current_time = datetime.now()
-for x in range(30):
-    # I know for a fact that sometimes reddit will only show 4 posts on the page,
-    # but each one of these options will only pull one of them. Just ask for all
-    # of them, smash them together, and process.
-    submissions = list(r.subreddit("transcribersofreddit").hot(limit=None))
-    submissions += list(r.subreddit("transcribersofreddit").new(limit=None))
-    submissions += list(r.subreddit("transcribersofreddit").top(limit=None))
-    submissions += list(r.subreddit("transcribersofreddit").controversial(limit=None))
-    remove_old_crap(submissions)
-
-    # try:
-    #     item.reply(
-    #         "This submission has been open for at least three days and is listed as in progress"
-    #         " -- it has been removed to make room for other submissions in the queue. Please contact"
-    #         " itsthejoker if there is an issue."
-    #     )
-    # except RedditAPIException:
-    #     pass
+
+if __name__ == '__main__':
+    current_time = datetime.now()
+    for x in range(30):
+        # I know for a fact that sometimes reddit will only show 4 posts on the page,
+        # but each one of these options will only pull one of them. Just ask for all
+        # of them, smash them together, and process.
+        submissions = list(r.subreddit("transcribersofreddit").hot(limit=None))
+        submissions += list(r.subreddit("transcribersofreddit").new(limit=None))
+        submissions += list(r.subreddit("transcribersofreddit").top(limit=None))
+        submissions += list(r.subreddit("transcribersofreddit").controversial(limit=None))
+        remove_old_crap(submissions)
+
+        # try:
+        #     item.reply(
+        #         "This submission has been open for at least three days and is listed as in progress"
+        #         " -- it has been removed to make room for other submissions in the queue. Please contact"
+        #         " itsthejoker if there is an issue."
+        #     )
+        # except RedditAPIException:
+        #     pass

test/core/test_admin_commands.py

@@ -1,8 +1,8 @@
-import pytest  # type: ignore
+import pytest
 from unittest.mock import MagicMock
 from unittest.mock import patch
 
-from tor.core.admin_commands import process_override
+from tor.core.admin_commands import process_override, is_moderator
 
 
 tor = MagicMock()
@@ -25,21 +25,21 @@ class reddit(MagicMock):
 
 def test_from_moderator_true():
     # enable dot notation to match what it's looking for
-    config = Object()
+    config = MagicMock()
     config.tor_mods = ['asdf', 'qwer']
-    reply = Object()
+    reply = MagicMock()
     reply.author = 'qwer'
 
-    assert from_moderator(reply, config) is True
+    assert is_moderator(reply.author, config) is True
 
 
 def test_from_moderator_false():
-    config = Object()
+    config = MagicMock()
     config.tor_mods = ['asdf', 'qwer']
-    reply = Object()
+    reply = MagicMock()
     reply.author = 'poiu'
 
-    assert from_moderator(reply, config) is False
+    assert is_moderator(reply, config) is False
 
 
 @pytest.mark.xfail(reason='Unmaintained test')
@@ -48,7 +48,7 @@ def test_from_moderator_false():
 def test_process_override_not_moderator(mock_clean_id, mock_process_done):
     # for use with anything that requires a reply object
 
-    config = Object()
+    config = MagicMock()
     config.no_gifs = ['asdf', 'qwer']
     config.tor_mods = ['asdf']
     config.r = reddit
@@ -62,12 +62,12 @@ def test_process_override_not_moderator(mock_clean_id, mock_process_done):
 
 
 @pytest.mark.skip(reason='Unfinished test implementation')
-@patch('tor.core.admin_commands.from_moderator', return_value=True)
+@patch('tor.core.admin_commands.is_moderator', return_value=True)
 @patch('tor.core.user_interaction.process_done')
-def test_process_override_not_moderator2(mock_process_done, asd):
+def test_process_override_not_moderator2(mock_process_done):
     # for use with anything that requires a reply object
 
-    config = Object()
+    config = MagicMock()
     config.no_gifs = ['asdf', 'qwer']
     config.tor_mods = ['asdf']
     config.r = reddit

tor/__init__.py

@@ -1,6 +1,12 @@
 import os
+import toml
+
+try:
+    with open(os.path.join(os.path.dirname(__file__), '..', 'pyproject.toml'), 'r') as f:
+        __version__ = toml.load(f)['tool']['poetry']['version']
+except OSError:
+    __version__ = 'unknown'
 
-__version__ = '4.2.4'
 __SELF_NAME__ = 'transcribersofreddit'
 __BOT_NAMES__ = os.environ.get('BOT_NAMES', 'transcribersofreddit,tor_archivist,transcribot').split(',')
 __root__ = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

tor/cli/main.py

@@ -1,10 +1,12 @@
 import argparse
+import atexit
 import os
 import time
 import logging
 
-from praw import Reddit  # type: ignore
+from praw import Reddit
 from dotenv import load_dotenv
+import beeline
 
 # The `import tor` lines is necessary because `tor.__SELF_NAME__` is
 # set here. Reason: https://gist.github.com/TheLonelyGhost/9dbe810c42d8f2edcf3388a8b19519e1
@@ -13,7 +15,7 @@
 from tor.core.config import config
 from tor.core.helpers import run_until_dead
 from tor.core.inbox import check_inbox
-from tor.core.initialize import configure_logging, initialize
+from tor.core.initialize import initialize
 from tor.helpers.flair import set_meta_flair_on_other_posts
 from tor.helpers.threaded_worker import threaded_check_submissions
 
@@ -103,6 +105,7 @@ def noop(cfg):
     pass
 
 
+@beeline.traced(name='run')
 def run(cfg):
     """
     Primary routine.
@@ -128,6 +131,23 @@ def main():
         datefmt='%Y-%m-%dT%H:%M:%S',
     )
 
+    honeycomb_key = os.getenv('HONEYCOMB_KEY', '')
+    if len(honeycomb_key) == 0:
+        # debug mode: print what would be sent to honeycomb.io to stderr
+        beeline.init(
+            writekey='',
+            dataset='transcribersofreddit',
+            service_name='tor_moderator',
+            debug=True
+        )
+    else:
+        beeline.init(
+            writekey=honeycomb_key,
+            dataset='transcribersofreddit',
+            service_name='tor_moderator'
+        )
+    atexit.register(beeline.close)
+
     config.debug_mode = opt.debug
 
     if config.debug_mode:

tor/core/admin_commands.py

@@ -3,12 +3,15 @@
 import random
 from typing import Dict
 
+import beeline
 from praw.models import Redditor
+
 from tor.core.helpers import _, clean_id, send_to_modchat
 from tor.core.initialize import initialize
 from tor.core.user_interaction import process_done
 
 
+@beeline.traced(name='process_command')
 def process_command(reply, cfg):
     """
     This function processes any commands send to the bot via PM with a subject
@@ -95,6 +98,7 @@ def is_moderator(username, cfg):
     return username in cfg.tor_mods
 
 
+@beeline.traced(name='process_override')
 def process_override(user: Redditor, blossom_submission: Dict, parent_id: str, cfg):
     """
     This process is for moderators of ToR to force u/transcribersofreddit

tor/core/config.py

@@ -1,24 +1,17 @@
 import datetime
 import os
 
+import bugsnag
 from blossom_wrapper import BlossomAPI
-from praw import Reddit  # type: ignore
-from praw.models import Subreddit  # type: ignore
-from praw.models.reddit.subreddit import ModeratorRelationship  # type: ignore
-from slackclient import SlackClient  # type: ignore
+from praw import Reddit
+from praw.models import Subreddit
+from praw.models.reddit.subreddit import ModeratorRelationship
+from slackclient import SlackClient
 from typing import Dict, List, Union
 
 from tor import __root__, __version__
 from tor.core import cached_property
 
-# Load configuration regardless of if bugsnag is setup correctly
-try:
-    import bugsnag  # type: ignore
-except ImportError:
-    # If loading from setup.py or bugsnag isn't installed, we
-    # don't want to bomb out completely
-    bugsnag = None
-
 
 class Config(object):
     """
@@ -52,10 +45,10 @@ class Config(object):
     @cached_property
     def blossom(self):
         return BlossomAPI(
-            email=os.getenv('BLOSSOM_EMAIL'),
-            password=os.getenv('BLOSSOM_PASSWORD'),
-            api_key=os.getenv('BLOSSOM_API_KEY'),
-            api_base_url=os.getenv('BLOSSOM_API_URL'),
+            email=os.environ['BLOSSOM_EMAIL'],
+            password=os.environ['BLOSSOM_PASSWORD'],
+            api_key=os.environ['BLOSSOM_API_KEY'],
+            api_base_url=os.environ['BLOSSOM_API_URL'],
         )
 
     @cached_property

tor/core/helpers.py

@@ -5,10 +5,11 @@
 import time
 from typing import List, Tuple
 
+import beeline
 from blossom_wrapper import BlossomStatus
-from praw.exceptions import APIException  # type: ignore
-from praw.models import Comment, Submission, Subreddit  # type: ignore
-from prawcore.exceptions import RequestException, ServerError, Forbidden, NotFound  # type: ignore
+from praw.exceptions import APIException
+from praw.models import Comment, Submission, Subreddit
+from prawcore.exceptions import RequestException, ServerError, Forbidden, NotFound
 
 import tor.core
 from tor.core import __version__
@@ -68,6 +69,7 @@ def clean_list(items: List[str]) -> List[str]:
     return list([item.strip() for item in items if item.strip()])
 
 
+@beeline.traced(name='send_to_modchat')
 def send_to_modchat(message: str, cfg: Config, channel='general') -> None:
     """
     Sends a message to the ToR mod chat.
@@ -138,6 +140,7 @@ def get_parent_post_id(post: Comment, subreddit: Subreddit) -> Submission:
         return subreddit.submission(id=clean_id(post.parent_id))
 
 
+@beeline.traced(name='get_wiki_page')
 def get_wiki_page(pagename: str, cfg: Config) -> str:
     """
     Return the contents of a given wiki page.
@@ -263,6 +266,7 @@ def _check_removal_required(submission: Submission, cfg: Config) -> Tuple[bool,
     return False, False
 
 
+@beeline.traced(name='remove_if_required')
 def remove_if_required(
     submission: Submission, blossom_id: str, cfg: Config
 ) -> Tuple[bool, bool]: