From 8c1986e6ef5f36a9006a8c5e13eb6c19253ca585 Mon Sep 17 00:00:00 2001 From: quartata Date: Thu, 15 Feb 2018 12:34:58 -0800 Subject: [PATCH 01/34] Deletionwatcher refactor --- chatcommunicate.py | 13 ++-- deletionwatcher.py | 158 ++++++++++++++++++++++++++------------------- ws.py | 3 +- 3 files changed, 97 insertions(+), 77 deletions(-) diff --git a/chatcommunicate.py b/chatcommunicate.py index 368094ea23..ee93f8b672 100644 --- a/chatcommunicate.py +++ b/chatcommunicate.py @@ -14,7 +14,6 @@ import yaml import datahandling -from deletionwatcher import DeletionWatcher from excepthook import log_exception from globalvars import GlobalVars from parsing import fetch_post_url_from_msg_content, fetch_owner_url_from_msg_content @@ -169,9 +168,9 @@ def send_messages(): _last_messages.reports.popitem(last=False) if room.deletion_watcher: - threading.Thread(name="deletion watcher", - target=DeletionWatcher.check_if_report_was_deleted, - args=(report_data[0], room.room._client.get_message(message_id))).start() + callback = room.room._client.get_message(message_id).delete + + GlobalVars.deletion_watcher.subscribe(report_data[0], callback=callback, max=120) _pickle_run.set() @@ -262,9 +261,9 @@ def tell_rooms(msg, has, hasnt, notify_site="", report_data=None): if room.block_time < timestamp and _global_block < timestamp: if report_data and "delay" in _room_roles and room_id in _room_roles["delay"]: - threading.Thread(name="delayed post", - target=DeletionWatcher.post_message_if_not_deleted, - args=(msg_pings, room, report_data)).start() + callback = lambda: _msg_queue.put((room, msg_pings, report_data)) + + GlobalVars.deletion_watcher.subscribe(report_data[0], callback=callback, max=300) else: _msg_queue.put((room, msg_pings, report_data)) diff --git a/deletionwatcher.py b/deletionwatcher.py index b8ab067ea9..fefca76e90 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -1,7 +1,10 @@ # coding=utf-8 import json +import os.path +import pickle import requests import time +import threading # noinspection PyPackageRequirements import websocket # noinspection PyPackageRequirements @@ -17,80 +20,99 @@ # noinspection PyClassHasNoInit,PyBroadException,PyMethodParameters class DeletionWatcher: - @classmethod - def update_site_id_list(self): - soup = BeautifulSoup(requests.get("https://meta.stackexchange.com/topbar/site-switcher/site-list").text, - "html.parser") - site_id_dict = {} - for site in soup.findAll("a", attrs={"data-id": True}): - site_name = urlparse(site["href"]).netloc - site_id = site["data-id"] - site_id_dict[site_name] = site_id - GlobalVars.site_id_dict = site_id_dict + def __init__(self): + DeletionWatcher.update_site_id_list() + + self.socket = websocket.create_connection("wss://qa.sockets.stackexchange.com/") + self.posts = {} + + if os.path.exists("deletionIDs.p"): + with open("deletionIDs.p", "rb") as fh: + for post in self._check_batch(pickle.load(fh)): + self.subscribe(post, pickle=False) + + self._save() + + threading.Thread(name="deletion watcher", target=self._start, daemon=True) + + def _start(self): + while True: + msg = self.socket.recv() + + if msg: + msg = json.loads(msg) + action = msg["action"] + + if action == "hb": + ws.send("hb") + else: + data = json.loads(msg)["data"] + + if data["a"] == "post-deleted": + try: + post_id, _, post_type, post_url, callback, max_time = self.posts[action["action"]] + + if not post_type == "answer" or ("aId" in d and str(d["aId"]) == post_id)): + self.socket.send("-" + action) + Tasks.do(metasmoke.Metasmoke.send_deletion_stats_for_post, post_url, True) + + if callback and (not max_time or time.time() < max_time): + callback() + except KeyError: + pass + + def subscribe(post_url, callback=None, pickle=True, timeout=300): + post_id, post_site, post_type = fetch_post_id_and_site_from_url(post_url) + + if post_site not in GlobalVars.site_id_dict: + return - @classmethod - def check_websocket_for_deletion(self, post_site_id, post_url, timeout): - time_to_check = time.time() + timeout - post_id = post_site_id[0] - post_type = post_site_id[2] if post_type == "answer": question_id = str(datahandling.get_post_site_id_link(post_site_id)) + if question_id is None: return else: question_id = post_id - post_site = post_site_id[1] - if post_site not in GlobalVars.site_id_dict: - return + site_id = GlobalVars.site_id_dict[post_site] + action = "{}-question-{}".format(site_id, question_id) + max_time = time.time() + timeout + + self.posts[action] = (post_id, post_site, post_type, post_url, callback, max_time) + self.socket.send(action) + + if pickle: + Tasks.do(self._save) + + def _save(): + pickle_output = {} + + for post_id, post_site, _, _, _, _ in self.posts.values(): + if post_site not in pickle_output: + pickle_output[post_site] = [post_id] + else: + pickle_output[post_site].append(post_id) + + with open("deletionIDs.p", "wb") as pickle_file: + pickle.dump(pickle_output, pickle_file) - ws = websocket.create_connection("wss://qa.sockets.stackexchange.com/") - ws.send(site_id + "-question-" + question_id) - - while time.time() < time_to_check: - ws.settimeout(time_to_check - time.time()) - try: - a = ws.recv() - except websocket.WebSocketTimeoutException: - Tasks.do(metasmoke.Metasmoke.send_deletion_stats_for_post, post_url, False) - return False - if a is not None and a != "": - try: - action = json.loads(a)["action"] - if action == "hb": - ws.send("hb") - continue - else: - d = json.loads(json.loads(a)["data"]) - except: - continue - if d["a"] == "post-deleted" and str(d["qId"]) == question_id: - if (post_type == "answer" and "aId" in d and str(d["aId"]) == post_id) or post_type == "question": - Tasks.do(metasmoke.Metasmoke.send_deletion_stats_for_post, post_url, True) - return True - - Tasks.do(metasmoke.Metasmoke.send_deletion_stats_for_post, post_url, False) - return False - - @classmethod - def check_if_report_was_deleted(self, post_url, message): - post_site_id = fetch_post_id_and_site_from_url(post_url) - was_report_deleted = self.check_websocket_for_deletion(post_site_id, post_url, 1200) - - if was_report_deleted: - try: - message.delete() - except: - pass - - @classmethod - def post_message_if_not_deleted(self, message_text, room, report_data): - post_url = report_data[0] - - post_site_id = fetch_post_id_and_site_from_url(post_url) - was_report_deleted = self.check_websocket_for_deletion(post_site_id, post_url, 300) - - if not was_report_deleted and not datahandling.is_false_positive(post_site_id[0:2]) and not \ - datahandling.is_ignored_post(post_site_id[0:2]): - - chatcommunicate._msg_queue.put((room, message_text, report_data)) + @staticmethod + def _check_batch(saved): + for site, posts in saved: + ids = ";".join([post_id for post_id in posts]) + uri = "https://api.stackexchange.com/2.2/posts/{}?site={}&key=IAkbitmze4B8KpacUfLqkw((".format(ids, site) + + for post in requests.get(uri).json()["items"]: + yield post["link"] + + @staticmethod + def update_site_id_list(): + soup = BeautifulSoup(requests.get("https://meta.stackexchange.com/topbar/site-switcher/site-list").text, + "html.parser") + site_id_dict = {} + for site in soup.findAll("a", attrs={"data-id": True}): + site_name = urlparse(site["href"]).netloc + site_id = site["data-id"] + site_id_dict[site_name] = site_id + GlobalVars.site_id_dict = site_id_dict diff --git a/ws.py b/ws.py index 10b26a4c87..be59f748e9 100644 --- a/ws.py +++ b/ws.py @@ -73,6 +73,7 @@ # We need an instance of bodyfetcher before load_files() is called GlobalVars.bodyfetcher = BodyFetcher() +GlobalVars.deletion_watcher = DeletionWatcher() load_files() filter_auto_ignored_posts() @@ -142,8 +143,6 @@ def restart_automatically(): log('info', GlobalVars.location) log('info', GlobalVars.metasmoke_host) -DeletionWatcher.update_site_id_list() - ws = websocket.create_connection("wss://qa.sockets.stackexchange.com/") ws.send("155-questions-active") From adc69c9381ce16ef728f8e5a9e4887720f1df52e Mon Sep 17 00:00:00 2001 From: quartata Date: Thu, 15 Feb 2018 12:41:19 -0800 Subject: [PATCH 02/34] Update deletionwatcher.py --- deletionwatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deletionwatcher.py b/deletionwatcher.py index fefca76e90..35ae82442e 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -28,7 +28,7 @@ def __init__(self): if os.path.exists("deletionIDs.p"): with open("deletionIDs.p", "rb") as fh: - for post in self._check_batch(pickle.load(fh)): + for post in DeletionWatcher._check_batch(pickle.load(fh)): self.subscribe(post, pickle=False) self._save() From 17b13a1434deeb2f872386a8f6e71e70b8322131 Mon Sep 17 00:00:00 2001 From: quartata Date: Thu, 15 Feb 2018 14:05:28 -0800 Subject: [PATCH 03/34] Multiple callbacks, always subscribe after spamhandling --- chatcommunicate.py | 4 ++-- deletionwatcher.py | 15 ++++++++++----- spamhandling.py | 2 ++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/chatcommunicate.py b/chatcommunicate.py index ee93f8b672..728b16637e 100644 --- a/chatcommunicate.py +++ b/chatcommunicate.py @@ -170,7 +170,7 @@ def send_messages(): if room.deletion_watcher: callback = room.room._client.get_message(message_id).delete - GlobalVars.deletion_watcher.subscribe(report_data[0], callback=callback, max=120) + GlobalVars.deletion_watcher.subscribe(report_data[0], callback=callback, timeout=120) _pickle_run.set() @@ -263,7 +263,7 @@ def tell_rooms(msg, has, hasnt, notify_site="", report_data=None): if report_data and "delay" in _room_roles and room_id in _room_roles["delay"]: callback = lambda: _msg_queue.put((room, msg_pings, report_data)) - GlobalVars.deletion_watcher.subscribe(report_data[0], callback=callback, max=300) + GlobalVars.deletion_watcher.subscribe(report_data[0], callback=callback, timeout=300) else: _msg_queue.put((room, msg_pings, report_data)) diff --git a/deletionwatcher.py b/deletionwatcher.py index 35ae82442e..e2d790ca0e 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -50,14 +50,15 @@ def _start(self): if data["a"] == "post-deleted": try: - post_id, _, post_type, post_url, callback, max_time = self.posts[action["action"]] + post_id, _, post_type, post_url, callbacks = self.posts[action["action"]] if not post_type == "answer" or ("aId" in d and str(d["aId"]) == post_id)): self.socket.send("-" + action) Tasks.do(metasmoke.Metasmoke.send_deletion_stats_for_post, post_url, True) - if callback and (not max_time or time.time() < max_time): - callback() + for callback, max_time in callbacks: + if callback and (not max_time or time.time() < max_time): + callback() except KeyError: pass @@ -79,8 +80,12 @@ def subscribe(post_url, callback=None, pickle=True, timeout=300): action = "{}-question-{}".format(site_id, question_id) max_time = time.time() + timeout - self.posts[action] = (post_id, post_site, post_type, post_url, callback, max_time) - self.socket.send(action) + if action in self.posts: + _, _, _, _, callbacks = self.posts[action] + callbacks.append((callback, max_time)) + else: + self.posts[action] = (post_id, post_site, post_type, post_url, [(callback, max_time)]) + self.socket.send(action) if pickle: Tasks.do(self._save) diff --git a/spamhandling.py b/spamhandling.py index 449c175690..1ac3367eb9 100644 --- a/spamhandling.py +++ b/spamhandling.py @@ -148,6 +148,8 @@ def handle_spam(post, reasons, why): else: chatcommunicate.tell_rooms(message, ("all", "site-" + post.post_site), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) + + GlobalVars.deletion_watcher.subscribe(post_url) except: exc_type, exc_obj, exc_tb = sys.exc_info() excepthook.uncaught_exception(exc_type, exc_obj, exc_tb) From 964d439a773dc61345665f0924ea9dc131f11f56 Mon Sep 17 00:00:00 2001 From: quartata Date: Thu, 15 Feb 2018 14:10:21 -0800 Subject: [PATCH 04/34] Final Flake: E999.WAD and IDIOTONIA.WAD --- chatcommunicate.py | 3 ++- deletionwatcher.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/chatcommunicate.py b/chatcommunicate.py index 728b16637e..8bcaa219ef 100644 --- a/chatcommunicate.py +++ b/chatcommunicate.py @@ -261,7 +261,8 @@ def tell_rooms(msg, has, hasnt, notify_site="", report_data=None): if room.block_time < timestamp and _global_block < timestamp: if report_data and "delay" in _room_roles and room_id in _room_roles["delay"]: - callback = lambda: _msg_queue.put((room, msg_pings, report_data)) + def callback(): + _msg_queue.put((room, msg_pings, report_data)) GlobalVars.deletion_watcher.subscribe(report_data[0], callback=callback, timeout=300) else: diff --git a/deletionwatcher.py b/deletionwatcher.py index e2d790ca0e..692504ffa1 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -44,7 +44,7 @@ def _start(self): action = msg["action"] if action == "hb": - ws.send("hb") + self.sock.send("hb") else: data = json.loads(msg)["data"] @@ -52,7 +52,7 @@ def _start(self): try: post_id, _, post_type, post_url, callbacks = self.posts[action["action"]] - if not post_type == "answer" or ("aId" in d and str(d["aId"]) == post_id)): + if not post_type == "answer" or ("aId" in data and str(data["aId"]) == post_id): self.socket.send("-" + action) Tasks.do(metasmoke.Metasmoke.send_deletion_stats_for_post, post_url, True) @@ -62,14 +62,14 @@ def _start(self): except KeyError: pass - def subscribe(post_url, callback=None, pickle=True, timeout=300): + def subscribe(self, post_url, callback=None, pickle=True, timeout=300): post_id, post_site, post_type = fetch_post_id_and_site_from_url(post_url) if post_site not in GlobalVars.site_id_dict: return if post_type == "answer": - question_id = str(datahandling.get_post_site_id_link(post_site_id)) + question_id = str(datahandling.get_post_site_id_link((post_id, post_site, post_type))) if question_id is None: return @@ -90,7 +90,7 @@ def subscribe(post_url, callback=None, pickle=True, timeout=300): if pickle: Tasks.do(self._save) - def _save(): + def _save(self): pickle_output = {} for post_id, post_site, _, _, _, _ in self.posts.values(): From 43cba159fed97de9e5724a1c5674b6dd42ad2ef8 Mon Sep 17 00:00:00 2001 From: quartata Date: Thu, 15 Feb 2018 15:41:05 -0800 Subject: [PATCH 05/34] fixes --- deletionwatcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deletionwatcher.py b/deletionwatcher.py index 692504ffa1..6bb8491f55 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -93,7 +93,7 @@ def subscribe(self, post_url, callback=None, pickle=True, timeout=300): def _save(self): pickle_output = {} - for post_id, post_site, _, _, _, _ in self.posts.values(): + for post_id, post_site, _, _, _ in self.posts.values(): if post_site not in pickle_output: pickle_output[post_site] = [post_id] else: @@ -104,7 +104,7 @@ def _save(self): @staticmethod def _check_batch(saved): - for site, posts in saved: + for site, posts in saved.items(): ids = ";".join([post_id for post_id in posts]) uri = "https://api.stackexchange.com/2.2/posts/{}?site={}&key=IAkbitmze4B8KpacUfLqkw((".format(ids, site) From 8ab4a4fbe8f928aacac919593f2904d361cc61fb Mon Sep 17 00:00:00 2001 From: quartata Date: Thu, 15 Feb 2018 15:48:38 -0800 Subject: [PATCH 06/34] cull FPs --- deletionwatcher.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deletionwatcher.py b/deletionwatcher.py index 6bb8491f55..4ae3c627ea 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -105,12 +105,17 @@ def _save(self): @staticmethod def _check_batch(saved): for site, posts in saved.items(): - ids = ";".join([post_id for post_id in posts]) + ids = ";".join([post_id for post_id in posts if not DeletionWatcher._ignore((post_id, site))]) uri = "https://api.stackexchange.com/2.2/posts/{}?site={}&key=IAkbitmze4B8KpacUfLqkw((".format(ids, site) for post in requests.get(uri).json()["items"]: yield post["link"] + @staticmethod + def _ignore(post_site_id): + return datahandling.is_false_positive(post_site_id) or datahandling.is_ignored_post(post_site_id) or \ + datahandling.is_auto_ignored_post(post_site_id) + @staticmethod def update_site_id_list(): soup = BeautifulSoup(requests.get("https://meta.stackexchange.com/topbar/site-switcher/site-list").text, From 726dba8d60b1f2eeec239729b2115c6ae6c9eaf1 Mon Sep 17 00:00:00 2001 From: quartata Date: Fri, 16 Feb 2018 17:40:12 -0800 Subject: [PATCH 07/34] Various fixes --- chatcommands.py | 5 +++++ chatcommunicate.py | 6 +++--- deletionwatcher.py | 13 +++++++------ tasks.py | 12 +++++++++--- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/chatcommands.py b/chatcommands.py index 7e224df2ec..2a72df6aa0 100644 --- a/chatcommands.py +++ b/chatcommands.py @@ -663,6 +663,11 @@ def inqueue(url): return "Not in queue." +@command() +def listening(): + return "{} post(s) currently monitored for deletion.".format(len(GlobalVars.deletion_watcher.posts)) + + # noinspection PyIncorrectDocstring,PyProtectedMember @command(str, whole_msg=True, privileged=True, arity=(0, 1)) def stappit(msg, location_search): diff --git a/chatcommunicate.py b/chatcommunicate.py index 8bcaa219ef..6ec18cd5f9 100644 --- a/chatcommunicate.py +++ b/chatcommunicate.py @@ -17,6 +17,7 @@ from excepthook import log_exception from globalvars import GlobalVars from parsing import fetch_post_url_from_msg_content, fetch_owner_url_from_msg_content +from tasks import Tasks LastMessages = collections.namedtuple("LastMessages", ["messages", "reports"]) @@ -261,10 +262,9 @@ def tell_rooms(msg, has, hasnt, notify_site="", report_data=None): if room.block_time < timestamp and _global_block < timestamp: if report_data and "delay" in _room_roles and room_id in _room_roles["delay"]: - def callback(): - _msg_queue.put((room, msg_pings, report_data)) + task = Tasks.later(_msg_queue.put, (room, msg_pings, report_data), after=300) - GlobalVars.deletion_watcher.subscribe(report_data[0], callback=callback, timeout=300) + GlobalVars.deletion_watcher.subscribe(report_data[0], callback=task.cancel) else: _msg_queue.put((room, msg_pings, report_data)) diff --git a/deletionwatcher.py b/deletionwatcher.py index 4ae3c627ea..7889a12e1d 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -33,7 +33,7 @@ def __init__(self): self._save() - threading.Thread(name="deletion watcher", target=self._start, daemon=True) + threading.Thread(name="deletion watcher", target=self._start, daemon=True).start() def _start(self): while True: @@ -44,13 +44,14 @@ def _start(self): action = msg["action"] if action == "hb": - self.sock.send("hb") + self.socket.send("hb") else: - data = json.loads(msg)["data"] + data = json.loads(msg["data"]) if data["a"] == "post-deleted": try: - post_id, _, post_type, post_url, callbacks = self.posts[action["action"]] + post_id, _, post_type, post_url, callbacks = self.posts[action] + del self.posts[action["action"]] if not post_type == "answer" or ("aId" in data and str(data["aId"]) == post_id): self.socket.send("-" + action) @@ -62,7 +63,7 @@ def _start(self): except KeyError: pass - def subscribe(self, post_url, callback=None, pickle=True, timeout=300): + def subscribe(self, post_url, callback=None, pickle=True, timeout=None): post_id, post_site, post_type = fetch_post_id_and_site_from_url(post_url) if post_site not in GlobalVars.site_id_dict: @@ -78,7 +79,7 @@ def subscribe(self, post_url, callback=None, pickle=True, timeout=300): site_id = GlobalVars.site_id_dict[post_site] action = "{}-question-{}".format(site_id, question_id) - max_time = time.time() + timeout + max_time = (time.time() + timeout) if timeout else None if action in self.posts: _, _, _, _, callbacks = self.posts[action] diff --git a/tasks.py b/tasks.py index d503ef2c71..2c8b5ca8ab 100644 --- a/tasks.py +++ b/tasks.py @@ -16,14 +16,18 @@ def _run(cls): @classmethod def do(cls, func, *args, **kwargs): - cls.loop.call_soon(lambda: func(*args, **kwargs)) + handle = cls.loop.call_soon(lambda: func(*args, **kwargs)) cls.loop._write_to_self() + return handle + @classmethod def later(cls, func, *args, after=None, **kwargs): - cls.loop.call_later(after, lambda: func(*args, **kwargs)) + handle = cls.loop.call_later(after, lambda: func(*args, **kwargs)) cls.loop._write_to_self() + return handle + @classmethod def periodic(cls, func, *args, interval=None, **kwargs): @asyncio.coroutine @@ -32,8 +36,10 @@ def f(): yield from asyncio.sleep(interval) func(*args, **kwargs) - cls.loop.create_task(f()) + handle = cls.loop.create_task(f()) cls.loop._write_to_self() + return handle + threading.Thread(name="tasks", target=Tasks._run, daemon=True).start() From bf24c4c0b2466f4476dcdbb09e62768a96354f6d Mon Sep 17 00:00:00 2001 From: quartata Date: Wed, 28 Feb 2018 15:27:00 -0800 Subject: [PATCH 08/34] log on missing site --- deletionwatcher.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deletionwatcher.py b/deletionwatcher.py index 7889a12e1d..7808eee51d 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -14,6 +14,7 @@ import metasmoke from globalvars import GlobalVars import datahandling +from helpers import log from parsing import fetch_post_id_and_site_from_url from tasks import Tasks @@ -67,6 +68,7 @@ def subscribe(self, post_url, callback=None, pickle=True, timeout=None): post_id, post_site, post_type = fetch_post_id_and_site_from_url(post_url) if post_site not in GlobalVars.site_id_dict: + log("warning", "unknown site {} when subscribing to {}".format(post_site, post_url) return if post_type == "answer": From 60b0c9a7b9a60296cd16365d13fe10fd9ca466a3 Mon Sep 17 00:00:00 2001 From: quartata Date: Wed, 28 Feb 2018 16:59:31 -0800 Subject: [PATCH 09/34] missed a parens there --- deletionwatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deletionwatcher.py b/deletionwatcher.py index 7808eee51d..8b172e1b28 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -68,7 +68,7 @@ def subscribe(self, post_url, callback=None, pickle=True, timeout=None): post_id, post_site, post_type = fetch_post_id_and_site_from_url(post_url) if post_site not in GlobalVars.site_id_dict: - log("warning", "unknown site {} when subscribing to {}".format(post_site, post_url) + log("warning", "unknown site {} when subscribing to {}".format(post_site, post_url)) return if post_type == "answer": From bea24af85bda0667013ade6074d07b78e563ce65 Mon Sep 17 00:00:00 2001 From: micsthepick Date: Tue, 13 Mar 2018 09:12:46 +1100 Subject: [PATCH 10/34] fix crash on targeted blame Now picks a random user on error --- chatcommands.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/chatcommands.py b/chatcommands.py index 786cec4b83..0c7cba942c 100644 --- a/chatcommands.py +++ b/chatcommands.py @@ -361,7 +361,10 @@ def blame2(msg, x): for i, char in enumerate(reversed(x)): user += (len(base)**i) * base[char] - unlucky_victim = msg._client.get_user(user) + try: + unlucky_victim = msg._client.get_user(user) + except HTTPError: + unlucky_victim = msg._client.get_user(random.choice(msg.room.get_current_user_ids())) return "It's [{}](https://chat.{}/users/{})'s fault.".format(unlucky_victim.name, msg._client.host, unlucky_victim.id) From bbe2418878f35d913be11913e648c5154a837c92 Mon Sep 17 00:00:00 2001 From: micsthepick Date: Tue, 13 Mar 2018 09:20:09 +1100 Subject: [PATCH 11/34] Remove extra whitespace after except --- chatcommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatcommands.py b/chatcommands.py index 0c7cba942c..47d989e1de 100644 --- a/chatcommands.py +++ b/chatcommands.py @@ -363,7 +363,7 @@ def blame2(msg, x): try: unlucky_victim = msg._client.get_user(user) - except HTTPError: + except HTTPError: unlucky_victim = msg._client.get_user(random.choice(msg.room.get_current_user_ids())) return "It's [{}](https://chat.{}/users/{})'s fault.".format(unlucky_victim.name, msg._client.host, From d0ce8afb2dd26d6b7cbbca13195fc99bf6427e97 Mon Sep 17 00:00:00 2001 From: micsthepick Date: Tue, 13 Mar 2018 09:42:27 +1100 Subject: [PATCH 12/34] Blame user if invalid targeted blame --- chatcommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatcommands.py b/chatcommands.py index 47d989e1de..1555d5dc8a 100644 --- a/chatcommands.py +++ b/chatcommands.py @@ -364,7 +364,7 @@ def blame2(msg, x): try: unlucky_victim = msg._client.get_user(user) except HTTPError: - unlucky_victim = msg._client.get_user(random.choice(msg.room.get_current_user_ids())) + unlucky_victim = msg.owner return "It's [{}](https://chat.{}/users/{})'s fault.".format(unlucky_victim.name, msg._client.host, unlucky_victim.id) From 62c36b1b379b922465474a654a2cd3489fb568fe Mon Sep 17 00:00:00 2001 From: micsthepick Date: Tue, 13 Mar 2018 10:14:52 +1100 Subject: [PATCH 13/34] move code that causes error into try block The previous pull was not correct --- chatcommands.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/chatcommands.py b/chatcommands.py index 1555d5dc8a..5bb9a50db4 100644 --- a/chatcommands.py +++ b/chatcommands.py @@ -363,11 +363,15 @@ def blame2(msg, x): try: unlucky_victim = msg._client.get_user(user) + return "It's [{}](https://chat.{}/users/{})'s fault.".format(unlucky_victim.name, + msg._client.host, + unlucky_victim.id) except HTTPError: unlucky_victim = msg.owner - return "It's [{}](https://chat.{}/users/{})'s fault.".format(unlucky_victim.name, + return "It's [{}](https://chat.{}/users/{})'s fault.".format(unlucky_victim.name, msg._client.host, unlucky_victim.id) + # noinspection PyIncorrectDocstring From f4b46fcbf38e68e1cea34638042b785488240cbb Mon Sep 17 00:00:00 2001 From: micsthepick Date: Tue, 13 Mar 2018 10:21:45 +1100 Subject: [PATCH 14/34] fix formatting issues --- chatcommands.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/chatcommands.py b/chatcommands.py index 5bb9a50db4..5cdfe2679c 100644 --- a/chatcommands.py +++ b/chatcommands.py @@ -364,14 +364,13 @@ def blame2(msg, x): try: unlucky_victim = msg._client.get_user(user) return "It's [{}](https://chat.{}/users/{})'s fault.".format(unlucky_victim.name, - msg._client.host, - unlucky_victim.id) + msg._client.host, + unlucky_victim.id) except HTTPError: unlucky_victim = msg.owner return "It's [{}](https://chat.{}/users/{})'s fault.".format(unlucky_victim.name, - msg._client.host, - unlucky_victim.id) - + msg._client.host, + unlucky_victim.id) # noinspection PyIncorrectDocstring From a135389458062f8661463d535d7005d54a09add2 Mon Sep 17 00:00:00 2001 From: quartata Date: Mon, 12 Mar 2018 16:46:43 -0700 Subject: [PATCH 15/34] action is already parsed --- deletionwatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deletionwatcher.py b/deletionwatcher.py index 8b172e1b28..d4a72e6697 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -52,7 +52,7 @@ def _start(self): if data["a"] == "post-deleted": try: post_id, _, post_type, post_url, callbacks = self.posts[action] - del self.posts[action["action"]] + del self.posts[action] if not post_type == "answer" or ("aId" in data and str(data["aId"]) == post_id): self.socket.send("-" + action) From 854acbdd5a6d35f7d3078506cbb38bd37dd74c86 Mon Sep 17 00:00:00 2001 From: SmokeDetector Date: Tue, 13 Mar 2018 02:11:17 +0000 Subject: [PATCH 16/34] Auto watch of plasticbidet\.com by WELZ --autopull --- watched_keywords.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/watched_keywords.txt b/watched_keywords.txt index 84523937a2..42847c3fd8 100644 --- a/watched_keywords.txt +++ b/watched_keywords.txt @@ -2599,3 +2599,4 @@ 1520861335 tripleee ^Nastia Mykoliuk$ 1520861497 tripleee ora-error\.com 1520882503 Glorfindel howtogethelpinwindows10\.co +1520907077 WELZ plasticbidet\.com From 12bc7bae339d935960dfe7067143873ab3be386d Mon Sep 17 00:00:00 2001 From: SmokeDetector Date: Tue, 13 Mar 2018 02:12:32 +0000 Subject: [PATCH 17/34] Auto watch of aquatownbidet\.com by WELZ --autopull --- watched_keywords.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/watched_keywords.txt b/watched_keywords.txt index 42847c3fd8..1baf6accb4 100644 --- a/watched_keywords.txt +++ b/watched_keywords.txt @@ -2600,3 +2600,4 @@ 1520861497 tripleee ora-error\.com 1520882503 Glorfindel howtogethelpinwindows10\.co 1520907077 WELZ plasticbidet\.com +1520907152 WELZ aquatownbidet\.com From fcbf83faf7b869578a9e508a39b9d4896a4a43c5 Mon Sep 17 00:00:00 2001 From: quartata Date: Mon, 12 Mar 2018 19:34:27 -0700 Subject: [PATCH 18/34] dump the whole post dict for debugging --- chatcommands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chatcommands.py b/chatcommands.py index 30658cba20..3a4b1c9134 100644 --- a/chatcommands.py +++ b/chatcommands.py @@ -672,8 +672,8 @@ def inqueue(url): @command() def listening(): - return "{} post(s) currently monitored for deletion.".format(len(GlobalVars.deletion_watcher.posts)) - + # return "{} post(s) currently monitored for deletion.".format(len(GlobalVars.deletion_watcher.posts)) + return repr(GlobalVars.deletion_watcher.posts) # noinspection PyIncorrectDocstring,PyProtectedMember @command(str, whole_msg=True, privileged=True, arity=(0, 1)) From 41b590fa572d6c9c73508f499e9ce7733fadc237 Mon Sep 17 00:00:00 2001 From: quartata Date: Mon, 12 Mar 2018 19:37:15 -0700 Subject: [PATCH 19/34] flake --- chatcommands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/chatcommands.py b/chatcommands.py index 3a4b1c9134..9d8116d3f9 100644 --- a/chatcommands.py +++ b/chatcommands.py @@ -675,6 +675,7 @@ def listening(): # return "{} post(s) currently monitored for deletion.".format(len(GlobalVars.deletion_watcher.posts)) return repr(GlobalVars.deletion_watcher.posts) + # noinspection PyIncorrectDocstring,PyProtectedMember @command(str, whole_msg=True, privileged=True, arity=(0, 1)) def stappit(msg, location_search): From 4ad654e5ed3c8e55302f0cf6d76b87d521790fe2 Mon Sep 17 00:00:00 2001 From: quartata Date: Mon, 12 Mar 2018 20:10:12 -0700 Subject: [PATCH 20/34] format post URLs correctly --- deletionwatcher.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deletionwatcher.py b/deletionwatcher.py index d4a72e6697..cf673a9242 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -83,11 +83,11 @@ def subscribe(self, post_url, callback=None, pickle=True, timeout=None): action = "{}-question-{}".format(site_id, question_id) max_time = (time.time() + timeout) if timeout else None - if action in self.posts: + if action in self.posts and callback: _, _, _, _, callbacks = self.posts[action] callbacks.append((callback, max_time)) else: - self.posts[action] = (post_id, post_site, post_type, post_url, [(callback, max_time)]) + self.posts[action] = (post_id, post_site, post_type, post_url, [(callback, max_time)] if callback else []) self.socket.send(action) if pickle: @@ -112,7 +112,7 @@ def _check_batch(saved): uri = "https://api.stackexchange.com/2.2/posts/{}?site={}&key=IAkbitmze4B8KpacUfLqkw((".format(ids, site) for post in requests.get(uri).json()["items"]: - yield post["link"] + yield "//" + post["link"] @staticmethod def _ignore(post_site_id): From 77cdd41a91a23a0bb93cc5ebd2590e80cf21bab2 Mon Sep 17 00:00:00 2001 From: quartata Date: Mon, 12 Mar 2018 20:15:45 -0700 Subject: [PATCH 21/34] use to_protocol_relative --- deletionwatcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deletionwatcher.py b/deletionwatcher.py index cf673a9242..a0a09c3290 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -15,7 +15,7 @@ from globalvars import GlobalVars import datahandling from helpers import log -from parsing import fetch_post_id_and_site_from_url +from parsing import fetch_post_id_and_site_from_url, to_protocol_relative from tasks import Tasks @@ -112,7 +112,7 @@ def _check_batch(saved): uri = "https://api.stackexchange.com/2.2/posts/{}?site={}&key=IAkbitmze4B8KpacUfLqkw((".format(ids, site) for post in requests.get(uri).json()["items"]: - yield "//" + post["link"] + yield to_protocol_relative(post["link"]) @staticmethod def _ignore(post_site_id): From 0f4af0e791d54ee5e5445c3a7050bd4221de7250 Mon Sep 17 00:00:00 2001 From: quartata Date: Mon, 12 Mar 2018 20:18:48 -0700 Subject: [PATCH 22/34] =?UTF-8?q?don=E2=80=99t=20need=20that=20check=20any?= =?UTF-8?q?more?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deletionwatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deletionwatcher.py b/deletionwatcher.py index a0a09c3290..3ac33b5706 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -59,7 +59,7 @@ def _start(self): Tasks.do(metasmoke.Metasmoke.send_deletion_stats_for_post, post_url, True) for callback, max_time in callbacks: - if callback and (not max_time or time.time() < max_time): + if not max_time or time.time() < max_time: callback() except KeyError: pass From e63ff83f69133cea143d4b4e6823baf77d83d903 Mon Sep 17 00:00:00 2001 From: quartata Date: Mon, 12 Mar 2018 20:41:27 -0700 Subject: [PATCH 23/34] dont stringify question ID --- deletionwatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deletionwatcher.py b/deletionwatcher.py index 3ac33b5706..e3e8fb4d5e 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -72,7 +72,7 @@ def subscribe(self, post_url, callback=None, pickle=True, timeout=None): return if post_type == "answer": - question_id = str(datahandling.get_post_site_id_link((post_id, post_site, post_type))) + question_id = datahandling.get_post_site_id_link((post_id, post_site, post_type)) if question_id is None: return From b369a9808ce0fd43f58bc839369dca56c968c66b Mon Sep 17 00:00:00 2001 From: quartata Date: Mon, 12 Mar 2018 20:43:57 -0700 Subject: [PATCH 24/34] unused tavern junk --- spamhandling.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/spamhandling.py b/spamhandling.py index 1ac3367eb9..f38bd8f53f 100644 --- a/spamhandling.py +++ b/spamhandling.py @@ -25,12 +25,6 @@ def should_whitelist_prevent_alert(user_url, reasons): return len(reasons_comparison) == 0 -# noinspection PyMissingTypeHints -def should_reasons_prevent_tavern_posting(reasons): - reasons_comparison = [r for r in set(reasons) if r not in GlobalVars.non_tavern_reasons] - return len(reasons_comparison) == 0 - - # noinspection PyMissingTypeHints def check_if_spam(post): # if not post.body: From 01dbc6b65cccfb6337d3244361c41785c54b643d Mon Sep 17 00:00:00 2001 From: quartata Date: Mon, 12 Mar 2018 21:04:37 -0700 Subject: [PATCH 25/34] cull posts older than 2 hours --- deletionwatcher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deletionwatcher.py b/deletionwatcher.py index e3e8fb4d5e..6a147435e0 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -112,7 +112,8 @@ def _check_batch(saved): uri = "https://api.stackexchange.com/2.2/posts/{}?site={}&key=IAkbitmze4B8KpacUfLqkw((".format(ids, site) for post in requests.get(uri).json()["items"]: - yield to_protocol_relative(post["link"]) + if time.time() - post["creation_date"] < 7200: + yield to_protocol_relative(post["link"]) @staticmethod def _ignore(post_site_id): From d28cc012250b8f8bd9e2ab557b591d010acc0532 Mon Sep 17 00:00:00 2001 From: quartata Date: Mon, 12 Mar 2018 21:22:00 -0700 Subject: [PATCH 26/34] check for FP --- chatcommunicate.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/chatcommunicate.py b/chatcommunicate.py index a2412e5a3b..0d4767e868 100644 --- a/chatcommunicate.py +++ b/chatcommunicate.py @@ -16,7 +16,7 @@ import datahandling from excepthook import log_exception from globalvars import GlobalVars -from parsing import fetch_post_url_from_msg_content, fetch_owner_url_from_msg_content +from parsing import fetch_post_id_and_site_from_url, fetch_post_url_from_msg_content, fetch_owner_url_from_msg_content from tasks import Tasks LastMessages = collections.namedtuple("LastMessages", ["messages", "reports"]) @@ -262,7 +262,13 @@ def tell_rooms(msg, has, hasnt, notify_site="", report_data=None): if room.block_time < timestamp and _global_block < timestamp: if report_data and "delay" in _room_roles and room_id in _room_roles["delay"]: - task = Tasks.later(_msg_queue.put, (room, msg_pings, report_data), after=300) + def callback(): + post = fetch_post_id_and_site_from_url(report_data[0])[0:2] + + if not datahandling.is_false_positive(post) and not datahandling.is_ignored_post(post): + _msg_queue.put(room, msg_pings, report_data) + + task = Tasks.later(callback, after=300) GlobalVars.deletion_watcher.subscribe(report_data[0], callback=task.cancel) else: From d699cc1cf39e672b972e37e701676f867a42c223 Mon Sep 17 00:00:00 2001 From: quartata Date: Mon, 12 Mar 2018 21:35:06 -0700 Subject: [PATCH 27/34] appease Metasmoke --- deletionwatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deletionwatcher.py b/deletionwatcher.py index 6a147435e0..32760df359 100644 --- a/deletionwatcher.py +++ b/deletionwatcher.py @@ -113,7 +113,7 @@ def _check_batch(saved): for post in requests.get(uri).json()["items"]: if time.time() - post["creation_date"] < 7200: - yield to_protocol_relative(post["link"]) + yield to_protocol_relative(post["link"]).replace("/q/", "/questions/") @staticmethod def _ignore(post_site_id): From 2c2c2fa0df1aa3e84d76cee790a18644593d4af2 Mon Sep 17 00:00:00 2001 From: SmokeDetector Date: Tue, 13 Mar 2018 05:08:50 +0000 Subject: [PATCH 28/34] Auto watch of lts\W?secure by tripleee --autopull --- watched_keywords.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/watched_keywords.txt b/watched_keywords.txt index 1baf6accb4..0549dfd087 100644 --- a/watched_keywords.txt +++ b/watched_keywords.txt @@ -2601,3 +2601,4 @@ 1520882503 Glorfindel howtogethelpinwindows10\.co 1520907077 WELZ plasticbidet\.com 1520907152 WELZ aquatownbidet\.com +1520917730 tripleee lts\W?secure From 99ad35b59e5cc7fe6e9a6ed0d32eb7ecfb75a927 Mon Sep 17 00:00:00 2001 From: SmokeDetector Date: Tue, 13 Mar 2018 05:09:29 +0000 Subject: [PATCH 29/34] Auto blacklist of nuallura by tripleee --autopull --- bad_keywords.txt | 1 + watched_keywords.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/bad_keywords.txt b/bad_keywords.txt index c0d8981a01..fd2d6897fe 100644 --- a/bad_keywords.txt +++ b/bad_keywords.txt @@ -1177,3 +1177,4 @@ d\W?bal mega\W?boost\W?perform\W?xl seo\W*services vidhigra +nuallura diff --git a/watched_keywords.txt b/watched_keywords.txt index 0549dfd087..287595563f 100644 --- a/watched_keywords.txt +++ b/watched_keywords.txt @@ -2584,7 +2584,6 @@ 1520772093 WELZ noavarangermi\.ir 1520773183 WELZ sourcesara\.com 1520795181 Glorfindel meshkalla -1520822770 iBug nuallura 1520830900 K.Dᴀᴠɪs chinasunlead\.net 1520834947 micsthepick ajk\Wtorus 1520837032 tripleee theitsol\.com From 984023ed4336a80237b1da31c7607b7c93489e54 Mon Sep 17 00:00:00 2001 From: SmokeDetector Date: Tue, 13 Mar 2018 05:12:13 +0000 Subject: [PATCH 30/34] Auto blacklist of goo\.gl/7J79so by tripleee --autopull --- blacklisted_websites.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/blacklisted_websites.txt b/blacklisted_websites.txt index fdad125df2..8dc4bd220e 100644 --- a/blacklisted_websites.txt +++ b/blacklisted_websites.txt @@ -1779,3 +1779,4 @@ healthcareorder\.com healthsuppfacts\.com adviksoft\.com skincare4your\.com +goo\.gl/7J79so From 187dfeee2543434a5d7af3ad4bc6895f1785ccca Mon Sep 17 00:00:00 2001 From: SmokeDetector Date: Tue, 13 Mar 2018 05:25:29 +0000 Subject: [PATCH 31/34] Auto watch of (?:7\W*)?914\W*611\W*04\W*44 by tripleee --autopull --- watched_keywords.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/watched_keywords.txt b/watched_keywords.txt index 287595563f..57683a4416 100644 --- a/watched_keywords.txt +++ b/watched_keywords.txt @@ -2601,3 +2601,4 @@ 1520907077 WELZ plasticbidet\.com 1520907152 WELZ aquatownbidet\.com 1520917730 tripleee lts\W?secure +1520918729 tripleee (?:7\W*)?914\W*611\W*04\W*44 From 1aa0862f928fe2192744518116337ffa43e950b0 Mon Sep 17 00:00:00 2001 From: SmokeDetector Date: Tue, 13 Mar 2018 05:26:20 +0000 Subject: [PATCH 32/34] Auto blacklist of 3peartechnologies\.com by tripleee --autopull --- blacklisted_websites.txt | 1 + watched_keywords.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/blacklisted_websites.txt b/blacklisted_websites.txt index 8dc4bd220e..58f41b0dc3 100644 --- a/blacklisted_websites.txt +++ b/blacklisted_websites.txt @@ -1780,3 +1780,4 @@ healthsuppfacts\.com adviksoft\.com skincare4your\.com goo\.gl/7J79so +3peartechnologies\.com diff --git a/watched_keywords.txt b/watched_keywords.txt index 57683a4416..e4ffebac37 100644 --- a/watched_keywords.txt +++ b/watched_keywords.txt @@ -2474,7 +2474,6 @@ 1519739799 Federico dentoaviation\.com 1519745836 Glorfindel ind99info\.com 1519786348 WELZ fix\W?card\W?tech -1519796009 tripleee 3peartechnologies\.com 1519802723 tripleee asetsafety\.ac\.in 1519806308 doppelgreener softsolution\.al 1519808512 Glorfindel hot166\.com From a94556e6d4ec71f2cde94d9763efc60299a524d7 Mon Sep 17 00:00:00 2001 From: SmokeDetector Date: Tue, 13 Mar 2018 05:31:43 +0000 Subject: [PATCH 33/34] Auto watch of golden\W?farms by tripleee --autopull --- watched_keywords.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/watched_keywords.txt b/watched_keywords.txt index e4ffebac37..3630db9b4b 100644 --- a/watched_keywords.txt +++ b/watched_keywords.txt @@ -2601,3 +2601,4 @@ 1520907152 WELZ aquatownbidet\.com 1520917730 tripleee lts\W?secure 1520918729 tripleee (?:7\W*)?914\W*611\W*04\W*44 +1520919103 tripleee golden\W?farms From 098bbad7fa04321655f9623a5c17992899714ebd Mon Sep 17 00:00:00 2001 From: SmokeDetector Date: Tue, 13 Mar 2018 05:33:29 +0000 Subject: [PATCH 34/34] Auto watch of rose\W?diamond by tripleee --autopull --- watched_keywords.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/watched_keywords.txt b/watched_keywords.txt index 3630db9b4b..008781467f 100644 --- a/watched_keywords.txt +++ b/watched_keywords.txt @@ -2602,3 +2602,4 @@ 1520917730 tripleee lts\W?secure 1520918729 tripleee (?:7\W*)?914\W*611\W*04\W*44 1520919103 tripleee golden\W?farms +1520919209 tripleee rose\W?diamond