From eadd1516669fb463f60ab6dbcf62f3a5d6235842 Mon Sep 17 00:00:00 2001 From: David Alexander Date: Sat, 5 Jun 2021 21:34:53 -0400 Subject: [PATCH 1/9] Updates allowed python version to be used --- poetry.lock | 54 ++++++-------------------------------------------- pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 49 deletions(-) diff --git a/poetry.lock b/poetry.lock index e80a9eb0..78f7cae7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -61,18 +61,18 @@ d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [[package]] name = "blossom-wrapper" -version = "0.0.1" -description = "Wrapper for the Blossom API" +version = "0.1.0" +description = "" category = "main" optional = false -python-versions = ">=3.6" +python-versions = "^3.8" develop = false [package.source] type = "git" url = "https://github.com/GrafeasGroup/blossom-wrapper.git" reference = "master" -resolved_reference = "d92361f9ae93ac620d50dcf71c2fe301c9aeae67" +resolved_reference = "7ccf68a363122da2ab934d19c988d331c85ba00d" [[package]] name = "bugsnag" @@ -115,7 +115,6 @@ python-versions = ">=3.6" [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [[package]] name = "colorama" @@ -145,7 +144,6 @@ optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [package.dependencies] -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} mccabe = ">=0.6.0,<0.7.0" pycodestyle = ">=2.7.0,<2.8.0" pyflakes = ">=2.3.0,<2.4.0" @@ -158,22 +156,6 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -[[package]] -name = "importlib-metadata" -version = "4.5.0" -description = "Read metadata from Python packages" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} -zipp = ">=0.5" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] - [[package]] name = "mccabe" version = "0.6.1" @@ -241,9 +223,6 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -[package.dependencies] -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} - [package.extras] dev = ["pre-commit", "tox"] @@ -328,7 +307,6 @@ python-versions = ">=3.5" atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} attrs = ">=17.4.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} more-itertools = ">=4.0.0" packaging = "*" pluggy = ">=0.12,<1.0" @@ -510,25 +488,13 @@ category = "main" optional = false python-versions = ">=3.6" -[[package]] -name = "zipp" -version = "3.4.1" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "pytest-enabler", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] - [extras] ci = [] [metadata] lock-version = "1.1" -python-versions = "^3.6" -content-hash = "8e43b045260f398790de9a4a4dce235107d3a8b348e58f5d947484be5600de43" +python-versions = "^3.9" +content-hash = "e8db93a2fe7431727d75b3904fcc858b4205ae455fa5b05d306da76f9e4730e0" [metadata.files] appdirs = [ @@ -634,10 +600,6 @@ idna = [ {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, ] -importlib-metadata = [ - {file = "importlib_metadata-4.5.0-py3-none-any.whl", hash = "sha256:833b26fb89d5de469b24a390e9df088d4e52e4ba33b01dc5e0e4f41b81a16c00"}, - {file = "importlib_metadata-4.5.0.tar.gz", hash = "sha256:b142cc1dd1342f31ff04bb7d022492b09920cb64fed867cd3ea6f80fe3ebd139"}, -] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, @@ -864,7 +826,3 @@ websocket-client = [ {file = "websocket-client-1.0.1.tar.gz", hash = "sha256:3e2bf58191d4619b161389a95bdce84ce9e0b24eb8107e7e590db682c2d0ca81"}, {file = "websocket_client-1.0.1-py2.py3-none-any.whl", hash = "sha256:abf306dc6351dcef07f4d40453037e51cc5d9da2ef60d0fc5d0fe3bcda255372"}, ] -zipp = [ - {file = "zipp-3.4.1-py3-none-any.whl", hash = "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098"}, - {file = "zipp-3.4.1.tar.gz", hash = "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76"}, -] diff --git a/pyproject.toml b/pyproject.toml index 464fc517..13996e03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ exclude = ['test', 'test.*', '*.test.*', '*.test'] include = ["commands.json", "tor/strings/*.yml"] [tool.poetry.dependencies] -python = "^3.6" +python = "^3.9" sh = "^1.12" bugsnag = "^3.6" requests = "^2.22" From f9446d211d0325d8cefe9e4ed422364f28ca7c1b Mon Sep 17 00:00:00 2001 From: David Alexander Date: Sat, 5 Jun 2021 21:35:26 -0400 Subject: [PATCH 2/9] Pulls bot version from pyproject.toml attribute --- poetry.lock | 4 ++-- pyproject.toml | 1 + tor/__init__.py | 8 +++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 78f7cae7..018f2803 100644 --- a/poetry.lock +++ b/poetry.lock @@ -411,7 +411,7 @@ websocket-client = "*" name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" @@ -494,7 +494,7 @@ ci = [] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "e8db93a2fe7431727d75b3904fcc858b4205ae455fa5b05d306da76f9e4730e0" +content-hash = "79e024cf1843faa0e66df7f401f3fc8ad6604fc4178b23a2ad9fd9e979b1a7c4" [metadata.files] appdirs = [ diff --git a/pyproject.toml b/pyproject.toml index 13996e03..b73c0107 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ PyYaml = "^5.1" blossom-wrapper = { git = "https://github.com/GrafeasGroup/blossom-wrapper.git", branch = "master" } python-dotenv = "^0.14.0" praw = "^7.1.0" +toml = "^0.10.2" [tool.poetry.dev-dependencies] better-exceptions = "^0.2.2" diff --git a/tor/__init__.py b/tor/__init__.py index 5cb3122f..b91b6f68 100644 --- a/tor/__init__.py +++ b/tor/__init__.py @@ -1,6 +1,12 @@ import os +import toml + +try: + with open(os.path.join(os.path.dirname(__file__), '..', 'pyproject.toml'), 'r') as f: + __version__ = toml.load(f)['tool']['poetry']['version'] +except OSError: + __version__ = 'unknown' -__version__ = '4.2.4' __SELF_NAME__ = 'transcribersofreddit' __BOT_NAMES__ = os.environ.get('BOT_NAMES', 'transcribersofreddit,tor_archivist,transcribot').split(',') __root__ = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) From 42a38b027383e19ef1c417512099b9764a4c4cf1 Mon Sep 17 00:00:00 2001 From: David Alexander Date: Sat, 5 Jun 2021 21:41:06 -0400 Subject: [PATCH 3/9] Removes unused import --- tor/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tor/cli/main.py b/tor/cli/main.py index 29092472..4cd75be7 100644 --- a/tor/cli/main.py +++ b/tor/cli/main.py @@ -13,7 +13,7 @@ from tor.core.config import config from tor.core.helpers import run_until_dead from tor.core.inbox import check_inbox -from tor.core.initialize import configure_logging, initialize +from tor.core.initialize import initialize from tor.helpers.flair import set_meta_flair_on_other_posts from tor.helpers.threaded_worker import threaded_check_submissions From 5665f1d554c845bc31fbe05b418dca735b99cde1 Mon Sep 17 00:00:00 2001 From: David Alexander Date: Sun, 6 Jun 2021 02:25:28 -0400 Subject: [PATCH 4/9] Adds Honeycomb tracing --- poetry.lock | 58 +++++++++++++++++++++++++++++++++- pyproject.toml | 1 + tor/cli/main.py | 20 ++++++++++++ tor/core/admin_commands.py | 4 +++ tor/core/helpers.py | 4 +++ tor/core/inbox.py | 6 +++- tor/core/initialize.py | 8 +++++ tor/core/posts.py | 3 ++ tor/core/user_interaction.py | 6 ++++ tor/helpers/threaded_worker.py | 30 +++++++++++------- 10 files changed, 126 insertions(+), 14 deletions(-) diff --git a/poetry.lock b/poetry.lock index 018f2803..9ffdad3f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -148,6 +148,18 @@ mccabe = ">=0.6.0,<0.7.0" pycodestyle = ">=2.7.0,<2.8.0" pyflakes = ">=2.3.0,<2.4.0" +[[package]] +name = "honeycomb-beeline" +version = "2.17.0" +description = "Honeycomb library for easy instrumentation" +category = "main" +optional = false +python-versions = ">=2.7" + +[package.dependencies] +libhoney = ">=1.7.0" +wrapt = ">=1.12.1,<2.0.0" + [[package]] name = "idna" version = "2.10" @@ -156,6 +168,19 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "libhoney" +version = "1.10.0" +description = "Python library for sending data to Honeycomb" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +requests = ">=2.24.0,<3.0.0" +six = ">=1.15.0,<2.0.0" +statsd = ">=3.3.0,<4.0.0" + [[package]] name = "mccabe" version = "0.6.1" @@ -407,6 +432,14 @@ requests = "*" six = "*" websocket-client = "*" +[[package]] +name = "statsd" +version = "3.3.0" +description = "A simple statsd client." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "toml" version = "0.10.2" @@ -488,13 +521,21 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "wrapt" +version = "1.12.1" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = false +python-versions = "*" + [extras] ci = [] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "79e024cf1843faa0e66df7f401f3fc8ad6604fc4178b23a2ad9fd9e979b1a7c4" +content-hash = "828fc7626a142c69a41a6d3c913e5baf0a8a388cb73f38ee09b2959e86659c64" [metadata.files] appdirs = [ @@ -596,10 +637,18 @@ flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, ] +honeycomb-beeline = [ + {file = "honeycomb-beeline-2.17.0.tar.gz", hash = "sha256:9287f69afdf22998fdb1636376b3e5bb271e137d61b8ffaec1e2b6c1b2e8d690"}, + {file = "honeycomb_beeline-2.17.0-py2.py3-none-any.whl", hash = "sha256:3078b701b2f43b1997d0fcf0016ffc6740643bcf1e6f36605f5c7e1f9efa25d9"}, +] idna = [ {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, ] +libhoney = [ + {file = "libhoney-1.10.0-py2.py3-none-any.whl", hash = "sha256:a7ea3783b9ca011174b1c657887c56c9055028861bc59c5137abd0a4bf22e3d0"}, + {file = "libhoney-1.10.0.tar.gz", hash = "sha256:4322c4e3a08f3304843f52ce8b19372569cf294db13d86d2fa7b466fcd90c979"}, +] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, @@ -765,6 +814,10 @@ six = [ slackclient = [ {file = "slackclient-1.0.6.tar.gz", hash = "sha256:e8ef8cd38d4fa86af0a662a24116332f610d14257cda5ff13a6a5b199e836dda"}, ] +statsd = [ + {file = "statsd-3.3.0-py2.py3-none-any.whl", hash = "sha256:c610fb80347fca0ef62666d241bce64184bd7cc1efe582f9690e045c25535eaa"}, + {file = "statsd-3.3.0.tar.gz", hash = "sha256:e3e6db4c246f7c59003e51c9720a51a7f39a396541cb9b147ff4b14d15b5dd1f"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -826,3 +879,6 @@ websocket-client = [ {file = "websocket-client-1.0.1.tar.gz", hash = "sha256:3e2bf58191d4619b161389a95bdce84ce9e0b24eb8107e7e590db682c2d0ca81"}, {file = "websocket_client-1.0.1-py2.py3-none-any.whl", hash = "sha256:abf306dc6351dcef07f4d40453037e51cc5d9da2ef60d0fc5d0fe3bcda255372"}, ] +wrapt = [ + {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"}, +] diff --git a/pyproject.toml b/pyproject.toml index b73c0107..950d62dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ blossom-wrapper = { git = "https://github.com/GrafeasGroup/blossom-wrapper.git", python-dotenv = "^0.14.0" praw = "^7.1.0" toml = "^0.10.2" +honeycomb-beeline = "^2.17.0" [tool.poetry.dev-dependencies] better-exceptions = "^0.2.2" diff --git a/tor/cli/main.py b/tor/cli/main.py index 4cd75be7..9f02f25d 100644 --- a/tor/cli/main.py +++ b/tor/cli/main.py @@ -1,10 +1,12 @@ import argparse +import atexit import os import time import logging from praw import Reddit # type: ignore from dotenv import load_dotenv +import beeline # The `import tor` lines is necessary because `tor.__SELF_NAME__` is # set here. Reason: https://gist.github.com/TheLonelyGhost/9dbe810c42d8f2edcf3388a8b19519e1 @@ -103,6 +105,7 @@ def noop(cfg): pass +@beeline.traced(name='run') def run(cfg): """ Primary routine. @@ -128,6 +131,23 @@ def main(): datefmt='%Y-%m-%dT%H:%M:%S', ) + honeycomb_key = os.getenv('HONEYCOMB_KEY', '') + if len(honeycomb_key) == 0: + # debug mode: print what would be sent to honeycomb.io to stderr + beeline.init( + writekey='', + dataset='transcribersofreddit', + service_name='tor_moderator', + debug=True + ) + else: + beeline.init( + writekey=honeycomb_key, + dataset='transcribersofreddit', + service_name='tor_moderator' + ) + atexit.register(beeline.close) + config.debug_mode = opt.debug if config.debug_mode: diff --git a/tor/core/admin_commands.py b/tor/core/admin_commands.py index 1333594d..65d6b838 100644 --- a/tor/core/admin_commands.py +++ b/tor/core/admin_commands.py @@ -3,12 +3,15 @@ import random from typing import Dict +import beeline from praw.models import Redditor + from tor.core.helpers import _, clean_id, send_to_modchat from tor.core.initialize import initialize from tor.core.user_interaction import process_done +@beeline.traced(name='process_command') def process_command(reply, cfg): """ This function processes any commands send to the bot via PM with a subject @@ -95,6 +98,7 @@ def is_moderator(username, cfg): return username in cfg.tor_mods +@beeline.traced(name='process_override') def process_override(user: Redditor, blossom_submission: Dict, parent_id: str, cfg): """ This process is for moderators of ToR to force u/transcribersofreddit diff --git a/tor/core/helpers.py b/tor/core/helpers.py index c1f2ffca..d4f7b27c 100644 --- a/tor/core/helpers.py +++ b/tor/core/helpers.py @@ -5,6 +5,7 @@ import time from typing import List, Tuple +import beeline from blossom_wrapper import BlossomStatus from praw.exceptions import APIException # type: ignore from praw.models import Comment, Submission, Subreddit # type: ignore @@ -68,6 +69,7 @@ def clean_list(items: List[str]) -> List[str]: return list([item.strip() for item in items if item.strip()]) +@beeline.traced(name='send_to_modchat') def send_to_modchat(message: str, cfg: Config, channel='general') -> None: """ Sends a message to the ToR mod chat. @@ -138,6 +140,7 @@ def get_parent_post_id(post: Comment, subreddit: Subreddit) -> Submission: return subreddit.submission(id=clean_id(post.parent_id)) +@beeline.traced(name='get_wiki_page') def get_wiki_page(pagename: str, cfg: Config) -> str: """ Return the contents of a given wiki page. @@ -263,6 +266,7 @@ def _check_removal_required(submission: Submission, cfg: Config) -> Tuple[bool, return False, False +@beeline.traced(name='remove_if_required') def remove_if_required( submission: Submission, blossom_id: str, cfg: Config ) -> Tuple[bool, bool]: diff --git a/tor/core/inbox.py b/tor/core/inbox.py index 43e79699..4af2f911 100644 --- a/tor/core/inbox.py +++ b/tor/core/inbox.py @@ -2,7 +2,7 @@ import random import re -from blossom_wrapper import BlossomStatus +import beeline from praw.exceptions import ClientException # type: ignore from praw.models import Comment, Message # type: ignore from praw.models.reddit.mixins import InboxableMixin # type: ignore @@ -29,6 +29,7 @@ log = logging.getLogger(__name__) +@beeline.traced(name='forward_to_slack') def forward_to_slack(item: InboxableMixin, cfg: Config) -> None: username = str(item.author.name) @@ -44,6 +45,7 @@ def forward_to_slack(item: InboxableMixin, cfg: Config) -> None: ) +@beeline.traced(name='process_reply') def process_reply(reply: Comment, cfg: Config) -> None: try: log.debug(f"Received reply from {reply.author.name}: {reply.body}") @@ -107,6 +109,7 @@ def process_reply(reply: Comment, cfg: Config) -> None: # uncommon, but common enough that this is necessary. +@beeline.traced(name='process_mention') def process_mention(mention: Comment) -> None: """ Handles username mentions and handles the formatting and posting of @@ -129,6 +132,7 @@ def process_mention(mention: Comment) -> None: pass +@beeline.traced(name='check_inbox') def check_inbox(cfg: Config) -> None: """ Goes through all the unread messages in the inbox. It deliberately diff --git a/tor/core/initialize.py b/tor/core/initialize.py index d764b8bc..ea2cca3b 100644 --- a/tor/core/initialize.py +++ b/tor/core/initialize.py @@ -1,5 +1,6 @@ import logging +import beeline from bugsnag.handlers import BugsnagHandler # type: ignore from tor.core.config import Config @@ -9,6 +10,7 @@ log = logging.getLogger() +@beeline.traced('configure_logging') def configure_logging(cfg: Config, log_name="transcribersofreddit.log") -> None: # Set formatting and logging level. logging.basicConfig( @@ -31,10 +33,12 @@ def configure_logging(cfg: Config, log_name="transcribersofreddit.log") -> None: log.info("*" * 50) +@beeline.traced('populate_header') def populate_header(cfg: Config) -> None: cfg.header = get_wiki_page('format/header', cfg) +@beeline.traced('populate_formatting') def populate_formatting(cfg: Config) -> None: """ Grabs the contents of the three wiki pages that contain the @@ -48,6 +52,7 @@ def populate_formatting(cfg: Config) -> None: cfg.other_formatting = get_wiki_page('format/other', cfg) +@beeline.traced('populate_domain_lists') def populate_domain_lists(cfg: Config) -> None: """ Loads the approved content domains into the config object from the @@ -74,6 +79,7 @@ def populate_domain_lists(cfg: Config) -> None: log.debug(f'Domain list populated: {current_domain_list}') +@beeline.traced('populate_subreddit_lists') def populate_subreddit_lists(cfg: Config) -> None: """ Gets the list of subreddits to monitor and loads it into memory. @@ -101,10 +107,12 @@ def populate_subreddit_lists(cfg: Config) -> None: log.debug(f'Retrieved subreddits subject to the upvote filter: {cfg.no_link_header_subs}') +@beeline.traced('populate_gifs') def populate_gifs(cfg: Config) -> None: cfg.no_gifs = get_wiki_page('usefulgifs/no', cfg).splitlines() +@beeline.traced('initialize') def initialize(cfg: Config) -> None: populate_domain_lists(cfg) log.debug('Domains loaded.') diff --git a/tor/core/posts.py b/tor/core/posts.py index 9c41ba64..743e96c9 100644 --- a/tor/core/posts.py +++ b/tor/core/posts.py @@ -1,6 +1,7 @@ import logging from typing import Dict, Union +import beeline from blossom_wrapper import BlossomStatus from praw.models import Submission # type: ignore @@ -17,6 +18,7 @@ PostSummary = Dict[str, Union[str, int, bool, None]] +@beeline.traced(name='process_post') def process_post(new_post: PostSummary, cfg: Config) -> None: """ After a valid post has been discovered, this handles the formatting @@ -118,6 +120,7 @@ def truncate_title(title: str) -> str: return title[:(max_length - 3)] + '...' +@beeline.traced(name='request_transcription') def request_transcription( post: PostSummary, content_type: str, content_format: str, cfg: Config ) -> None: diff --git a/tor/core/user_interaction.py b/tor/core/user_interaction.py index 4cc3d112..0bd4e283 100644 --- a/tor/core/user_interaction.py +++ b/tor/core/user_interaction.py @@ -2,6 +2,7 @@ import random from typing import Dict, Tuple +import beeline from blossom_wrapper import BlossomStatus from praw.models import Comment, Message, Redditor, Submission # type: ignore @@ -37,6 +38,7 @@ ] +@beeline.traced(name='process_coc') def process_coc( username: str, context: str, blossom_submission: Dict, cfg: Config ) -> Tuple: @@ -74,6 +76,7 @@ def process_coc( return process_claim(username, blossom_submission, cfg) +@beeline.traced(name='process_claim') def process_claim( username: str, blossom_submission: Dict, cfg: Config, first_time=False ) -> Tuple: @@ -115,6 +118,7 @@ def process_claim( return message, return_flair +@beeline.traced(name='process_done') def process_done( user: Redditor, blossom_submission: Dict, @@ -204,6 +208,7 @@ def process_done( return message, return_flair +@beeline.traced(name='process_unclaim') def process_unclaim( username: str, blossom_submission: Dict, submission: Submission, cfg: Config ) -> Tuple: @@ -248,6 +253,7 @@ def process_unclaim( return message, return_flair +@beeline.traced(name='process_message') def process_message(message: Message, cfg: Config) -> None: dm_subject = i18n['responses']['direct_message']['dm_subject'] dm_body = i18n['responses']['direct_message']['dm_body'] diff --git a/tor/helpers/threaded_worker.py b/tor/helpers/threaded_worker.py index 0353a6e4..a982c332 100644 --- a/tor/helpers/threaded_worker.py +++ b/tor/helpers/threaded_worker.py @@ -12,6 +12,7 @@ from datetime import datetime, timedelta from typing import Dict, List +import beeline import requests from tor.core.config import Config @@ -42,6 +43,7 @@ def check_domain_filter(item: Dict, cfg: Config) -> bool: return False +@beeline.traced_thread def get_subreddit_posts(sub: str) -> List[PostSummary]: def generate_user_agent() -> str: """ @@ -82,25 +84,29 @@ def parse_json_posts(posts: Dict) -> List[PostSummary]: }) return trimmed_links - headers = { - 'User-Agent': generate_user_agent() - } - url = f'https://www.reddit.com/r/{sub}/new/.json' - result = requests.get(url, headers=headers).json() - # we have two states here: one has the data we want and the other is an - # error state. The error state looks like this: - # {'message': 'Too Many Requests', 'error': 429} + with beeline.tracer(name='get_subreddit_posts'): + beeline.add_context({'subreddit', sub}) - if result.get('error', None): - log.warning('hit error state for {}'.format(sub)) - return [] - return parse_json_posts(result) + headers = { + 'User-Agent': generate_user_agent() + } + url = f'https://www.reddit.com/r/{sub}/new/.json' + result = requests.get(url, headers=headers).json() + # we have two states here: one has the data we want and the other is an + # error state. The error state looks like this: + # {'message': 'Too Many Requests', 'error': 429} + + if result.get('error', None): + log.warning('hit error state for {}'.format(sub)) + return [] + return parse_json_posts(result) def is_time_to_scan(cfg: Config) -> bool: return datetime.now() > cfg.last_post_scan_time + timedelta(seconds=45) +@beeline.traced(name='threaded_check_submissions') def threaded_check_submissions(cfg: Config) -> None: """ Single threaded PRAW performance: From 13d87f27f116e6eeb83a534a296390ceb09a0748 Mon Sep 17 00:00:00 2001 From: David Alexander Date: Sun, 6 Jun 2021 03:02:09 -0400 Subject: [PATCH 5/9] Support Python 3.9 --- .github/workflows/automated-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/automated-tests.yml b/.github/workflows/automated-tests.yml index 07d4d146..ebfd2476 100644 --- a/.github/workflows/automated-tests.yml +++ b/.github/workflows/automated-tests.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: '3.8' + python-version: '3.9' architecture: 'x64' - run: | pip install --upgrade pip @@ -33,7 +33,7 @@ jobs: max-parallel: 4 matrix: python-version: - - '3.8' + - '3.9' steps: - uses: actions/checkout@v2 From b7824eb4dd569cd55996a103da8eec9f5fa08a7c Mon Sep 17 00:00:00 2001 From: David Alexander Date: Sat, 12 Jun 2021 00:03:24 -0400 Subject: [PATCH 6/9] Fixes mypy issues --- poetry.lock | 88 ++++++++++++++++++++++++++------ pyproject.toml | 19 ++++++- test/core/test_admin_commands.py | 2 +- tor/cli/main.py | 2 +- tor/core/config.py | 26 ++++------ tor/core/helpers.py | 6 +-- tor/core/inbox.py | 6 +-- tor/core/initialize.py | 2 +- tor/core/posts.py | 2 +- tor/core/user_interaction.py | 2 +- tor/core/validation.py | 2 +- tor/helpers/flair.py | 2 +- tor/helpers/reddit_ids.py | 2 +- 13 files changed, 112 insertions(+), 49 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9ffdad3f..7a609d0c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -199,7 +199,7 @@ python-versions = ">=3.5" [[package]] name = "mypy" -version = "0.761" +version = "0.902" description = "Optional static typing for Python" category = "dev" optional = false @@ -207,11 +207,12 @@ python-versions = ">=3.5" [package.dependencies] mypy-extensions = ">=0.4.3,<0.5.0" -typed-ast = ">=1.4.0,<1.5.0" +toml = "*" typing-extensions = ">=3.7.4" [package.extras] dmypy = ["psutil (>=4.0)"] +python2 = ["typed-ast (>=1.4.0,<1.5.0)"] [[package]] name = "mypy-extensions" @@ -456,6 +457,38 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "types-pyyaml" +version = "0.1.6" +description = "Typing stubs for PyYAML" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "types-redis" +version = "3.5.0" +description = "Typing stubs for redis" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "types-requests" +version = "0.1.10" +description = "Typing stubs for requests" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "types-toml" +version = "0.1.1" +description = "Typing stubs for toml" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "typing-extensions" version = "3.10.0.0" @@ -535,7 +568,7 @@ ci = [] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "828fc7626a142c69a41a6d3c913e5baf0a8a388cb73f38ee09b2959e86659c64" +content-hash = "f13b0ff4211321e9375689b7c07be760d33c9ea4b7d43e13980e657a1c8dc2dc" [metadata.files] appdirs = [ @@ -658,20 +691,29 @@ more-itertools = [ {file = "more_itertools-8.8.0-py3-none-any.whl", hash = "sha256:2cf89ec599962f2ddc4d568a05defc40e0a587fbc10d5989713638864c36be4d"}, ] mypy = [ - {file = "mypy-0.761-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:7f672d02fffcbace4db2b05369142e0506cdcde20cea0e07c7c2171c4fd11dd6"}, - {file = "mypy-0.761-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:87c556fb85d709dacd4b4cb6167eecc5bbb4f0a9864b69136a0d4640fdc76a36"}, - {file = "mypy-0.761-cp35-cp35m-win_amd64.whl", hash = "sha256:c6d27bd20c3ba60d5b02f20bd28e20091d6286a699174dfad515636cb09b5a72"}, - {file = "mypy-0.761-cp36-cp36m-macosx_10_6_x86_64.whl", hash = "sha256:4b9365ade157794cef9685791032521233729cb00ce76b0ddc78749abea463d2"}, - {file = "mypy-0.761-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:634aef60b4ff0f650d3e59d4374626ca6153fcaff96ec075b215b568e6ee3cb0"}, - {file = "mypy-0.761-cp36-cp36m-win_amd64.whl", hash = "sha256:53ea810ae3f83f9c9b452582261ea859828a9ed666f2e1ca840300b69322c474"}, - {file = "mypy-0.761-cp37-cp37m-macosx_10_6_x86_64.whl", hash = "sha256:0a9a45157e532da06fe56adcfef8a74629566b607fa2c1ac0122d1ff995c748a"}, - {file = "mypy-0.761-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:7eadc91af8270455e0d73565b8964da1642fe226665dd5c9560067cd64d56749"}, - {file = "mypy-0.761-cp37-cp37m-win_amd64.whl", hash = "sha256:e2bb577d10d09a2d8822a042a23b8d62bc3b269667c9eb8e60a6edfa000211b1"}, - {file = "mypy-0.761-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c35cae79ceb20d47facfad51f952df16c2ae9f45db6cb38405a3da1cf8fc0a7"}, - {file = "mypy-0.761-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:f97a605d7c8bc2c6d1172c2f0d5a65b24142e11a58de689046e62c2d632ca8c1"}, - {file = "mypy-0.761-cp38-cp38-win_amd64.whl", hash = "sha256:a6bd44efee4dc8c3324c13785a9dc3519b3ee3a92cada42d2b57762b7053b49b"}, - {file = "mypy-0.761-py3-none-any.whl", hash = "sha256:7e396ce53cacd5596ff6d191b47ab0ea18f8e0ec04e15d69728d530e86d4c217"}, - {file = "mypy-0.761.tar.gz", hash = "sha256:85baab8d74ec601e86134afe2bcccd87820f79d2f8d5798c889507d1088287bf"}, + {file = "mypy-0.902-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:3f12705eabdd274b98f676e3e5a89f247ea86dc1af48a2d5a2b080abac4e1243"}, + {file = "mypy-0.902-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:2f9fedc1f186697fda191e634ac1d02f03d4c260212ccb018fabbb6d4b03eee8"}, + {file = "mypy-0.902-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:0756529da2dd4d53d26096b7969ce0a47997123261a5432b48cc6848a2cb0bd4"}, + {file = "mypy-0.902-cp35-cp35m-win_amd64.whl", hash = "sha256:68a098c104ae2b75e946b107ef69dd8398d54cb52ad57580dfb9fc78f7f997f0"}, + {file = "mypy-0.902-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cd01c599cf9f897b6b6c6b5d8b182557fb7d99326bcdf5d449a0fbbb4ccee4b9"}, + {file = "mypy-0.902-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:e89880168c67cf4fde4506b80ee42f1537ad66ad366c101d388b3fd7d7ce2afd"}, + {file = "mypy-0.902-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:ebe2bc9cb638475f5d39068d2dbe8ae1d605bb8d8d3ff281c695df1670ab3987"}, + {file = "mypy-0.902-cp36-cp36m-win_amd64.whl", hash = "sha256:f89bfda7f0f66b789792ab64ce0978e4a991a0e4dd6197349d0767b0f1095b21"}, + {file = "mypy-0.902-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:746e0b0101b8efec34902810047f26a8c80e1efbb4fc554956d848c05ef85d76"}, + {file = "mypy-0.902-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:0190fb77e93ce971954c9e54ea61de2802065174e5e990c9d4c1d0f54fbeeca2"}, + {file = "mypy-0.902-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:b5dfcd22c6bab08dfeded8d5b44bdcb68c6f1ab261861e35c470b89074f78a70"}, + {file = "mypy-0.902-cp37-cp37m-win_amd64.whl", hash = "sha256:b5ba1f0d5f9087e03bf5958c28d421a03a4c1ad260bf81556195dffeccd979c4"}, + {file = "mypy-0.902-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9ef5355eaaf7a23ab157c21a44c614365238a7bdb3552ec3b80c393697d974e1"}, + {file = "mypy-0.902-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:517e7528d1be7e187a5db7f0a3e479747307c1b897d9706b1c662014faba3116"}, + {file = "mypy-0.902-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:fd634bc17b1e2d6ce716f0e43446d0d61cdadb1efcad5c56ca211c22b246ebc8"}, + {file = "mypy-0.902-cp38-cp38-win_amd64.whl", hash = "sha256:fc4d63da57ef0e8cd4ab45131f3fe5c286ce7dd7f032650d0fbc239c6190e167"}, + {file = "mypy-0.902-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:353aac2ce41ddeaf7599f1c73fed2b75750bef3b44b6ad12985a991bc002a0da"}, + {file = "mypy-0.902-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ae94c31bb556ddb2310e4f913b706696ccbd43c62d3331cd3511caef466871d2"}, + {file = "mypy-0.902-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:8be7bbd091886bde9fcafed8dd089a766fa76eb223135fe5c9e9798f78023a20"}, + {file = "mypy-0.902-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:4efc67b9b3e2fddbe395700f91d5b8deb5980bfaaccb77b306310bd0b9e002eb"}, + {file = "mypy-0.902-cp39-cp39-win_amd64.whl", hash = "sha256:9f1d74eeb3f58c7bd3f3f92b8f63cb1678466a55e2c4612bf36909105d0724ab"}, + {file = "mypy-0.902-py3-none-any.whl", hash = "sha256:a26d0e53e90815c765f91966442775cf03b8a7514a4e960de7b5320208b07269"}, + {file = "mypy-0.902.tar.gz", hash = "sha256:9236c21194fde5df1b4d8ebc2ef2c1f2a5dc7f18bcbea54274937cae2e20a01c"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, @@ -854,6 +896,18 @@ typed-ast = [ {file = "typed_ast-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c"}, {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, ] +types-pyyaml = [ + {file = "types_PyYAML-0.1.6-py2.py3-none-any.whl", hash = "sha256:8620045462a0ae1ecff7e97041905abdedc7afdff7ea66cac31df29fb1af6fa3"}, +] +types-redis = [ + {file = "types_redis-3.5.0-py2.py3-none-any.whl", hash = "sha256:cf29f55ccc3b44c78259cc22e4c0030f77b7a47ec5656211402be0fcf1a98536"}, +] +types-requests = [ + {file = "types_requests-0.1.10-py2.py3-none-any.whl", hash = "sha256:5b1864f1564adb810123c563b4de86236c14058978ec1294b07a988c532cb079"}, +] +types-toml = [ + {file = "types_toml-0.1.1-py2.py3-none-any.whl", hash = "sha256:ccebb129f6d68f49fbeb0efd000d48098b3f62885f0dbe44ad001e8b9c0b62ff"}, +] typing-extensions = [ {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"}, {file = "typing_extensions-3.10.0.0-py3-none-any.whl", hash = "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"}, diff --git a/pyproject.toml b/pyproject.toml index 950d62dd..ab00dc69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,12 @@ better-exceptions = "^0.2.2" pytest = "^5.1" pytest-cov = "^2.7" flake8 = "^3.7.9" -mypy = "^0.761" +mypy = ">=0.800" black = "^19.10b0" +types-toml = "^0.1" +types-PyYAML = "^0.1" +types-requests = "^0.1" +types-redis = "^3.5" [tool.poetry.scripts] tor-moderator = "tor.cli.main:main" @@ -41,6 +45,19 @@ tor-moderator = "tor.cli.main:main" [tool.poetry.extras] ci = ['pytest', 'pytest-cov'] +[[tool.mypy.overrides]] +module = [ + "praw", + "praw.*", + "prawcore", + "prawcore.*", + "blossom_wrapper", + "beeline", + "slackclient", + "pytest", +] +ignore_missing_imports = true + [build-system] requires = ["poetry>=0.12"] build-backend = "poetry.masonry.api" diff --git a/test/core/test_admin_commands.py b/test/core/test_admin_commands.py index 569f26cf..2247a2f1 100644 --- a/test/core/test_admin_commands.py +++ b/test/core/test_admin_commands.py @@ -1,4 +1,4 @@ -import pytest # type: ignore +import pytest from unittest.mock import MagicMock from unittest.mock import patch diff --git a/tor/cli/main.py b/tor/cli/main.py index 9f02f25d..1c8e458b 100644 --- a/tor/cli/main.py +++ b/tor/cli/main.py @@ -4,7 +4,7 @@ import time import logging -from praw import Reddit # type: ignore +from praw import Reddit from dotenv import load_dotenv import beeline diff --git a/tor/core/config.py b/tor/core/config.py index 838649fc..eadee6dc 100644 --- a/tor/core/config.py +++ b/tor/core/config.py @@ -1,25 +1,17 @@ import datetime import os +import bugsnag from blossom_wrapper import BlossomAPI -from praw import Reddit # type: ignore -from praw.models import Subreddit # type: ignore -from praw.models.reddit.subreddit import ModeratorRelationship # type: ignore -from slackclient import SlackClient # type: ignore +from praw import Reddit +from praw.models import Subreddit +from praw.models.reddit.subreddit import ModeratorRelationship +from slackclient import SlackClient from typing import Dict, List, Union from tor import __root__, __version__ from tor.core import cached_property -# Load configuration regardless of if bugsnag is setup correctly -try: - import bugsnag # type: ignore -except ImportError: - # If loading from setup.py or bugsnag isn't installed, we - # don't want to bomb out completely - bugsnag = None - - class Config(object): """ A singleton object for checking global configuration from @@ -52,10 +44,10 @@ class Config(object): @cached_property def blossom(self): return BlossomAPI( - email=os.getenv('BLOSSOM_EMAIL'), - password=os.getenv('BLOSSOM_PASSWORD'), - api_key=os.getenv('BLOSSOM_API_KEY'), - api_base_url=os.getenv('BLOSSOM_API_URL'), + email=os.environ['BLOSSOM_EMAIL'], + password=os.environ['BLOSSOM_PASSWORD'], + api_key=os.environ['BLOSSOM_API_KEY'], + api_base_url=os.environ['BLOSSOM_API_URL'], ) @cached_property diff --git a/tor/core/helpers.py b/tor/core/helpers.py index d4f7b27c..90179937 100644 --- a/tor/core/helpers.py +++ b/tor/core/helpers.py @@ -7,9 +7,9 @@ import beeline from blossom_wrapper import BlossomStatus -from praw.exceptions import APIException # type: ignore -from praw.models import Comment, Submission, Subreddit # type: ignore -from prawcore.exceptions import RequestException, ServerError, Forbidden, NotFound # type: ignore +from praw.exceptions import APIException +from praw.models import Comment, Submission, Subreddit +from prawcore.exceptions import RequestException, ServerError, Forbidden, NotFound import tor.core from tor.core import __version__ diff --git a/tor/core/inbox.py b/tor/core/inbox.py index 4af2f911..8f566285 100644 --- a/tor/core/inbox.py +++ b/tor/core/inbox.py @@ -3,9 +3,9 @@ import re import beeline -from praw.exceptions import ClientException # type: ignore -from praw.models import Comment, Message # type: ignore -from praw.models.reddit.mixins import InboxableMixin # type: ignore +from praw.exceptions import ClientException +from praw.models import Comment, Message +from praw.models.reddit.mixins import InboxableMixin from tor import __BOT_NAMES__ from tor.core import validation diff --git a/tor/core/initialize.py b/tor/core/initialize.py index ea2cca3b..b9b55fd1 100644 --- a/tor/core/initialize.py +++ b/tor/core/initialize.py @@ -1,7 +1,7 @@ import logging import beeline -from bugsnag.handlers import BugsnagHandler # type: ignore +from bugsnag.handlers import BugsnagHandler from tor.core.config import Config from tor.core.helpers import clean_list, get_wiki_page diff --git a/tor/core/posts.py b/tor/core/posts.py index 743e96c9..c0192f70 100644 --- a/tor/core/posts.py +++ b/tor/core/posts.py @@ -3,7 +3,7 @@ import beeline from blossom_wrapper import BlossomStatus -from praw.models import Submission # type: ignore +from praw.models import Submission from tor.core.config import Config from tor.core.helpers import _ diff --git a/tor/core/user_interaction.py b/tor/core/user_interaction.py index 0bd4e283..2ca78b00 100644 --- a/tor/core/user_interaction.py +++ b/tor/core/user_interaction.py @@ -4,7 +4,7 @@ import beeline from blossom_wrapper import BlossomStatus -from praw.models import Comment, Message, Redditor, Submission # type: ignore +from praw.models import Comment, Message, Redditor, Submission from tor.core.config import Config from tor.core.helpers import (_, get_wiki_page, diff --git a/tor/core/validation.py b/tor/core/validation.py index fa8a4914..ec7c01fb 100644 --- a/tor/core/validation.py +++ b/tor/core/validation.py @@ -1,6 +1,6 @@ from typing import Union, Tuple -from praw.models import Comment, Redditor, Submission # type: ignore +from praw.models import Comment, Redditor, Submission from tor.core.config import Config from tor.strings import translation diff --git a/tor/helpers/flair.py b/tor/helpers/flair.py index 84b841f8..e087be96 100644 --- a/tor/helpers/flair.py +++ b/tor/helpers/flair.py @@ -1,7 +1,7 @@ import logging from blossom_wrapper import BlossomStatus -from praw.models import Comment, Redditor, Submission # type: ignore +from praw.models import Comment, Redditor, Submission from tor import __BOT_NAMES__ from tor.core.config import Config diff --git a/tor/helpers/reddit_ids.py b/tor/helpers/reddit_ids.py index 77f0a917..a15b8788 100644 --- a/tor/helpers/reddit_ids.py +++ b/tor/helpers/reddit_ids.py @@ -1,5 +1,5 @@ from blossom_wrapper import BlossomStatus -from praw.models import Submission # type: ignore +from praw.models import Submission from tor.core.config import Config From 4c8ab1ad9cd66b00e08cf7a25c69223329b5e83c Mon Sep 17 00:00:00 2001 From: David Alexander Date: Sat, 12 Jun 2021 00:29:05 -0400 Subject: [PATCH 7/9] Fixes automated tests --- test/core/test_admin_commands.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/core/test_admin_commands.py b/test/core/test_admin_commands.py index 2247a2f1..dda0c56d 100644 --- a/test/core/test_admin_commands.py +++ b/test/core/test_admin_commands.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock from unittest.mock import patch -from tor.core.admin_commands import process_override +from tor.core.admin_commands import process_override, is_moderator tor = MagicMock() @@ -25,21 +25,21 @@ class reddit(MagicMock): def test_from_moderator_true(): # enable dot notation to match what it's looking for - config = Object() + config = MagicMock() config.tor_mods = ['asdf', 'qwer'] - reply = Object() + reply = MagicMock() reply.author = 'qwer' - assert from_moderator(reply, config) is True + assert is_moderator(reply.author, config) is True def test_from_moderator_false(): - config = Object() + config = MagicMock() config.tor_mods = ['asdf', 'qwer'] - reply = Object() + reply = MagicMock() reply.author = 'poiu' - assert from_moderator(reply, config) is False + assert is_moderator(reply, config) is False @pytest.mark.xfail(reason='Unmaintained test') @@ -48,7 +48,7 @@ def test_from_moderator_false(): def test_process_override_not_moderator(mock_clean_id, mock_process_done): # for use with anything that requires a reply object - config = Object() + config = MagicMock() config.no_gifs = ['asdf', 'qwer'] config.tor_mods = ['asdf'] config.r = reddit @@ -62,12 +62,12 @@ def test_process_override_not_moderator(mock_clean_id, mock_process_done): @pytest.mark.skip(reason='Unfinished test implementation') -@patch('tor.core.admin_commands.from_moderator', return_value=True) +@patch('tor.core.admin_commands.is_moderator', return_value=True) @patch('tor.core.user_interaction.process_done') -def test_process_override_not_moderator2(mock_process_done, asd): +def test_process_override_not_moderator2(mock_process_done, mock_process_done): # for use with anything that requires a reply object - config = Object() + config = MagicMock() config.no_gifs = ['asdf', 'qwer'] config.tor_mods = ['asdf'] config.r = reddit From 0228732600fb608d1fd166476ccadda9abe19646 Mon Sep 17 00:00:00 2001 From: David Alexander Date: Sat, 12 Jun 2021 00:32:42 -0400 Subject: [PATCH 8/9] Fixes flake8 complaints --- scripts/clear_submissions_from_queue.py | 43 +++++++++++++------------ test/core/test_admin_commands.py | 2 +- tor/core/config.py | 1 + tor/core/user_interaction.py | 2 +- tor/core/validation.py | 2 +- tor/helpers/threaded_worker.py | 1 + 6 files changed, 28 insertions(+), 23 deletions(-) diff --git a/scripts/clear_submissions_from_queue.py b/scripts/clear_submissions_from_queue.py index c61a9752..c46f4c88 100644 --- a/scripts/clear_submissions_from_queue.py +++ b/scripts/clear_submissions_from_queue.py @@ -1,13 +1,14 @@ from datetime import datetime, timedelta from praw import Reddit -from praw.exceptions import RedditAPIException +# from praw.exceptions import RedditAPIException r = Reddit('bot2') UNCLAIMED = 'Unclaimed' IN_PROGRESS = 'In Progress' + def remove_old_crap(posts): for item in posts: if item.link_flair_text == UNCLAIMED: @@ -16,22 +17,24 @@ def remove_old_crap(posts): print(f"Removing {item.name}, posted on {str(submission_time)}") item.mod.remove() -current_time = datetime.now() -for x in range(30): - # I know for a fact that sometimes reddit will only show 4 posts on the page, - # but each one of these options will only pull one of them. Just ask for all - # of them, smash them together, and process. - submissions = list(r.subreddit("transcribersofreddit").hot(limit=None)) - submissions += list(r.subreddit("transcribersofreddit").new(limit=None)) - submissions += list(r.subreddit("transcribersofreddit").top(limit=None)) - submissions += list(r.subreddit("transcribersofreddit").controversial(limit=None)) - remove_old_crap(submissions) - - # try: - # item.reply( - # "This submission has been open for at least three days and is listed as in progress" - # " -- it has been removed to make room for other submissions in the queue. Please contact" - # " itsthejoker if there is an issue." - # ) - # except RedditAPIException: - # pass + +if __name__ == '__main__': + current_time = datetime.now() + for x in range(30): + # I know for a fact that sometimes reddit will only show 4 posts on the page, + # but each one of these options will only pull one of them. Just ask for all + # of them, smash them together, and process. + submissions = list(r.subreddit("transcribersofreddit").hot(limit=None)) + submissions += list(r.subreddit("transcribersofreddit").new(limit=None)) + submissions += list(r.subreddit("transcribersofreddit").top(limit=None)) + submissions += list(r.subreddit("transcribersofreddit").controversial(limit=None)) + remove_old_crap(submissions) + + # try: + # item.reply( + # "This submission has been open for at least three days and is listed as in progress" + # " -- it has been removed to make room for other submissions in the queue. Please contact" + # " itsthejoker if there is an issue." + # ) + # except RedditAPIException: + # pass diff --git a/test/core/test_admin_commands.py b/test/core/test_admin_commands.py index dda0c56d..fbf8cd37 100644 --- a/test/core/test_admin_commands.py +++ b/test/core/test_admin_commands.py @@ -64,7 +64,7 @@ def test_process_override_not_moderator(mock_clean_id, mock_process_done): @pytest.mark.skip(reason='Unfinished test implementation') @patch('tor.core.admin_commands.is_moderator', return_value=True) @patch('tor.core.user_interaction.process_done') -def test_process_override_not_moderator2(mock_process_done, mock_process_done): +def test_process_override_not_moderator2(mock_process_done): # for use with anything that requires a reply object config = MagicMock() diff --git a/tor/core/config.py b/tor/core/config.py index eadee6dc..76839aac 100644 --- a/tor/core/config.py +++ b/tor/core/config.py @@ -12,6 +12,7 @@ from tor import __root__, __version__ from tor.core import cached_property + class Config(object): """ A singleton object for checking global configuration from diff --git a/tor/core/user_interaction.py b/tor/core/user_interaction.py index 2ca78b00..ff7c47dc 100644 --- a/tor/core/user_interaction.py +++ b/tor/core/user_interaction.py @@ -7,7 +7,7 @@ from praw.models import Comment, Message, Redditor, Submission from tor.core.config import Config -from tor.core.helpers import (_, get_wiki_page, +from tor.core.helpers import (get_wiki_page, remove_if_required, send_to_modchat) from tor.core.validation import get_transcription from tor.helpers.flair import flair, set_user_flair diff --git a/tor/core/validation.py b/tor/core/validation.py index ec7c01fb..fbe014dd 100644 --- a/tor/core/validation.py +++ b/tor/core/validation.py @@ -1,6 +1,6 @@ from typing import Union, Tuple -from praw.models import Comment, Redditor, Submission +from praw.models import Comment, Redditor from tor.core.config import Config from tor.strings import translation diff --git a/tor/helpers/threaded_worker.py b/tor/helpers/threaded_worker.py index a982c332..cf93b46e 100644 --- a/tor/helpers/threaded_worker.py +++ b/tor/helpers/threaded_worker.py @@ -22,6 +22,7 @@ log = logging.getLogger() i18n = translation() + def check_domain_filter(item: Dict, cfg: Config) -> bool: """ Validate that a given post is actually one that we can (or should) work on From 8282d554e62f841562b05a2c644e17c1ca9ce4f6 Mon Sep 17 00:00:00 2001 From: David Alexander Date: Sat, 12 Jun 2021 01:08:05 -0400 Subject: [PATCH 9/9] Bumps dependency versions - prawcore (2.0.0 -> 2.2.0) - websocket-client (1.0.1 -> 0.54.0) - slackclient (1.0.6 -> 1.3.2) --- poetry.lock | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7a609d0c..8d71ae1b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -274,7 +274,7 @@ test = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.3.0,<0.5)", "pytest (>=2. [[package]] name = "prawcore" -version = "2.0.0" +version = "2.2.0" description = "Low-level communication layer for PRAW 4+." category = "main" optional = false @@ -422,16 +422,16 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "slackclient" -version = "1.0.6" -description = "Python client for Slack.com" +version = "1.3.2" +description = "Slack API clients for Web API and RTM API" category = "main" optional = false python-versions = "*" [package.dependencies] -requests = "*" -six = "*" -websocket-client = "*" +requests = ">=2.11,<3.0a0" +six = ">=1.10,<2.0a0" +websocket-client = ">=0.35,<0.55.0" [[package]] name = "statsd" @@ -548,11 +548,14 @@ testing = ["pytest (>=3.1.0)", "coverage", "pytest-cov", "pytest-xdist"] [[package]] name = "websocket-client" -version = "1.0.1" -description = "WebSocket client for Python with low level API options" +version = "0.54.0" +description = "WebSocket client for Python. hybi13 is supported." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +six = "*" [[package]] name = "wrapt" @@ -736,8 +739,8 @@ praw = [ {file = "praw-7.2.0.tar.gz", hash = "sha256:434600bb77264dc5f8f7635eb0541a6c992b87e1d9e9f6e3f40fe98e2c508152"}, ] prawcore = [ - {file = "prawcore-2.0.0-py3-none-any.whl", hash = "sha256:2353c0de28a01ccd39615f3ba0975d0781e4b51d2913f0965903dd98ebdd9a75"}, - {file = "prawcore-2.0.0.tar.gz", hash = "sha256:b498d9b6f54991005e727d5235c8f49ea5ba0c9a6d793fb743b40fa089cd36d1"}, + {file = "prawcore-2.2.0-py3-none-any.whl", hash = "sha256:cb7a01e43202233d62c4f9c0da890f6e6a534cebc16fb6ef1e71aff7077b0f42"}, + {file = "prawcore-2.2.0.tar.gz", hash = "sha256:bde42fad459c4dcfe0f22a18921ef4981ee7cd286ea1de3eb697ba91838c9123"}, ] py = [ {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, @@ -854,7 +857,7 @@ six = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] slackclient = [ - {file = "slackclient-1.0.6.tar.gz", hash = "sha256:e8ef8cd38d4fa86af0a662a24116332f610d14257cda5ff13a6a5b199e836dda"}, + {file = "slackclient-1.3.2.tar.gz", hash = "sha256:2623a76d9b94671d34c258366a1b726ba6e9499931bf1e515e4a61fea9855445"}, ] statsd = [ {file = "statsd-3.3.0-py2.py3-none-any.whl", hash = "sha256:c610fb80347fca0ef62666d241bce64184bd7cc1efe582f9690e045c25535eaa"}, @@ -930,8 +933,8 @@ webob = [ {file = "WebOb-1.8.7.tar.gz", hash = "sha256:b64ef5141be559cfade448f044fa45c2260351edcb6a8ef6b7e00c7dcef0c323"}, ] websocket-client = [ - {file = "websocket-client-1.0.1.tar.gz", hash = "sha256:3e2bf58191d4619b161389a95bdce84ce9e0b24eb8107e7e590db682c2d0ca81"}, - {file = "websocket_client-1.0.1-py2.py3-none-any.whl", hash = "sha256:abf306dc6351dcef07f4d40453037e51cc5d9da2ef60d0fc5d0fe3bcda255372"}, + {file = "websocket_client-0.54.0-py2.py3-none-any.whl", hash = "sha256:8c8bf2d4f800c3ed952df206b18c28f7070d9e3dcbd6ca6291127574f57ee786"}, + {file = "websocket_client-0.54.0.tar.gz", hash = "sha256:e51562c91ddb8148e791f0155fdb01325d99bb52c4cdbb291aee7a3563fd0849"}, ] wrapt = [ {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"},