diff --git a/newapi/DB_bots/pymysql_bot.py b/newapi/DB_bots/pymysql_bot.py index 7be0cc5..50386fc 100644 --- a/newapi/DB_bots/pymysql_bot.py +++ b/newapi/DB_bots/pymysql_bot.py @@ -4,11 +4,12 @@ """ import copy +import logging import pymysql import pymysql.cursors -from ..api_utils.except_err import exception_err +logger = logging.getLogger(__name__) def sql_connect_pymysql( @@ -24,7 +25,7 @@ def sql_connect_pymysql( try: connection = pymysql.connect(**args, **credentials) except Exception as e: - exception_err(e) + logger.exception(e) return [] with connection as conn, conn.cursor() as cursor: @@ -36,13 +37,14 @@ def sql_connect_pymysql( cursor.execute(query, params) except Exception as e: - exception_err(e) + logger.exception(e) return [] try: results = cursor.fetchall() except Exception as e: - exception_err(e) + logger.exception(e) + logger.exception('Exception during fetchall', exc_info=True) return [] return results diff --git a/newapi/api_utils/ask_bot.py b/newapi/api_utils/ask_bot.py index 3a99894..40e5769 100644 --- a/newapi/api_utils/ask_bot.py +++ b/newapi/api_utils/ask_bot.py @@ -3,21 +3,37 @@ from ...api_utils.ask_bot import ASK_BOT """ - import sys +import difflib -from . import printe +import logging +logger = logging.getLogger(__name__) yes_answer = ["y", "a", "", "Y", "A", "all", "aaa"] Save_or_Ask = {} +def showDiff(text, newtext): + diff = difflib.unified_diff(text.splitlines(), newtext.splitlines(), lineterm="") + for line in diff: + logger.info(line) + + class ASK_BOT: def __init__(self): pass - def ask_put(self, nodiff=False, newtext="", text="", message="", job="Genral", username="", summary=""): + def ask_put( + self, + nodiff=False, + newtext="", + text="", + message="", + job="Genral", + username="", + summary="", + ): """ Prompts the user to confirm saving changes to a page, optionally displaying a diff. @@ -36,29 +52,29 @@ def ask_put(self, nodiff=False, newtext="", text="", message="", job="Genral", u if text or newtext: if "nodiff" not in sys.argv and not nodiff: if len(newtext) < 70000 and len(text) < 70000 or "diff" in sys.argv: - printe.showDiff(text, newtext) + showDiff(text, newtext) else: - printe.output("showDiff error..") + logger.info("showDiff error..") # --- - printe.output(f"diference in bytes: {len(newtext) - len(text):,}") - printe.output(f"len of text: {len(text):,}, len of newtext: {len(newtext):,}") + logger.info(f"diference in bytes: {len(newtext) - len(text):,}") + logger.info(f"len of text: {len(text):,}, len of newtext: {len(newtext):,}") # --- if summary: - printe.output(f"-Edit summary: {summary}") + logger.info(f"-Edit summary: {summary}") # --- - printe.output(f"<>ASK_BOT: {message}? (yes, no) {username=}") + logger.info(f"<>ASK_BOT: {message}? (yes, no) {username=}") # --- sa = input("([y]es, [N]o, [a]ll)?") # --- if sa == "a": Save_or_Ask[job] = True # --- - printe.output("<> ---------------------------------") - printe.output(f"<> save all:{job} without asking.") - printe.output("<> ---------------------------------") + logger.info("<> ---------------------------------") + logger.info(f"<> save all:{job} without asking.") + logger.info("<> ---------------------------------") # --- if sa not in yes_answer: - printe.output("wrong answer") + logger.info("wrong answer") return False # --- return True diff --git a/newapi/api_utils/bot_edit/bot_edit_by_time.py b/newapi/api_utils/bot_edit/bot_edit_by_time.py index 6bce11a..092a067 100644 --- a/newapi/api_utils/bot_edit/bot_edit_by_time.py +++ b/newapi/api_utils/bot_edit/bot_edit_by_time.py @@ -1,9 +1,9 @@ """ """ import datetime +import logging -from .. import printe - +logger = logging.getLogger(__name__) Created_Cache = {} @@ -42,8 +42,8 @@ def check_create_time(page, title_page): wait_time = delay_hours - diff # --- if diff < delay_hours: - printe.output(f"<>Page:{title_page} create at ({create_time}).") - printe.output(f"<>Page Created before {diff:.2f} hours by: {user}, wait {wait_time:.2f}H.") + logger.info(f"<>Page:{title_page} create at ({create_time}).") + logger.info(f"<>Page Created before {diff:.2f} hours by: {user}, wait {wait_time:.2f}H.") return False # --- return True @@ -76,13 +76,13 @@ def check_last_edit_time(page, title_page, delay): # --- diff_minutes = (now - ts_time).total_seconds() / 60 # --- - # printe.output(f"<> last-edit Δ={diff_minutes:.2f} min for {title_page}") + # logger.info(f"<> last-edit Δ={diff_minutes:.2f} min for {title_page}") # --- wait_time = delay - diff_minutes # --- if diff_minutes < delay: - printe.output(f"<>Page:{title_page} last edit ({timestamp}).") - printe.output( + logger.info(f"<>Page:{title_page} last edit ({timestamp}).") + logger.info( f"<>Page Last edit before {delay} minutes, Wait {wait_time:.2f} minutes. title:{title_page}" ) return False diff --git a/newapi/api_utils/except_err.py b/newapi/api_utils/except_err.py index ecbf658..149ac37 100644 --- a/newapi/api_utils/except_err.py +++ b/newapi/api_utils/except_err.py @@ -1,20 +1,12 @@ """ - -python3 bots/new/newapi/except_err.py - -from .except_err import exception_err -from .except_err import exception_err, warn_err - """ import inspect +import logging import traceback from warnings import warn -try: - from . import printe -except Exception: - import printe +logger = logging.getLogger(__name__) def warn_err(err): @@ -36,17 +28,16 @@ def exception_err(e, text=""): if not isinstance(text, str): text = str(text) # --- - printe.output("<> start exception_err:") + logger.info("<> start :") # --- - printe.error("Traceback (most recent call last):") + logger.error("Traceback (most recent call last):") warn(warn_err(f"Exception:{str(e)}"), UserWarning, stacklevel=3) - printe.warn(text) + logger.warning(text) # --- if str(e) not in common_errors: # --- err = traceback.format_exc(limit=4) err = str(err).replace("Traceback (most recent call last):", "").strip() - printe.warn(err) + logger.warning(err) # --- - printe.warn("CRITICAL:") - # printe.info("====") + logger.critical("CRITICAL:") diff --git a/newapi/api_utils/printe.py b/newapi/api_utils/printe.py index e0da352..46531a1 100644 --- a/newapi/api_utils/printe.py +++ b/newapi/api_utils/printe.py @@ -1,12 +1,4 @@ """ -This module provides functions for printing colored text and showing differences between two texts. -The main functions are `output` and `showDiff`. - -Example usage: -# To print colored text -printe.output('<>red') # prints 'red' in red color -# To show differences between two texts -printe.showDiff('old text', 'new text') # prints the differences between 'old text' and 'new text' """ import difflib @@ -344,7 +336,10 @@ class Hunk: PENDING = 0 def __init__( - self, a: str | Sequence[str], b: str | Sequence[str], grouped_opcode: Sequence[tuple[str, int, int, int, int]] + self, + a: str | Sequence[str], + b: str | Sequence[str], + grouped_opcode: Sequence[tuple[str, int, int, int, int]], ) -> None: """ Initializer. @@ -539,7 +534,12 @@ def get_header_text(a_rng: tuple[int, int], b_rng: tuple[int, int], affix: str = class PatchManager: def __init__( - self, text_a: str, text_b: str, context: int = 0, by_letter: bool = False, replace_invisible: bool = False + self, + text_a: str, + text_b: str, + context: int = 0, + by_letter: bool = False, + replace_invisible: bool = False, ) -> None: self.a = text_a.splitlines(True) self.b = text_b.splitlines(True) @@ -633,8 +633,14 @@ def _get_context_range(self, super_hunk: _Superhunk) -> tuple[tuple[int, int], t a0, a1 = super_hunk.a_rng b0, b1 = super_hunk.b_rng return ( - (a0 - min(super_hunk.pre_context, self.context), a1 + min(super_hunk.post_context, self.context)), - (b0 - min(super_hunk.pre_context, self.context), b1 + min(super_hunk.post_context, self.context)), + ( + a0 - min(super_hunk.pre_context, self.context), + a1 + min(super_hunk.post_context, self.context), + ), + ( + b0 - min(super_hunk.pre_context, self.context), + b1 + min(super_hunk.post_context, self.context), + ), ) def _generate_diff(self, hunks: _Superhunk) -> str: diff --git a/newapi/api_utils/txtlib.py b/newapi/api_utils/txtlib.py index af67568..7b48ba1 100644 --- a/newapi/api_utils/txtlib.py +++ b/newapi/api_utils/txtlib.py @@ -11,8 +11,6 @@ # for temp in temps: name, namestrip, params, template = temp['name'], temp['namestrip'], temp['params'], temp['item'] """ -# from ..other_bots import printe - from functools import lru_cache import wikitextparser as wtp @@ -42,7 +40,7 @@ def extract_templates_and_params(text): # --- name = str(template.normal_name()).strip() pa_item = template.string - # printe.output( "<> pa_item: %s" % pa_item ) + # logger.info( "<> pa_item: %s" % pa_item ) # --- namestrip = name # --- diff --git a/newapi/api_utils/user_agent.py b/newapi/api_utils/user_agent.py index 62b9a0b..804c384 100644 --- a/newapi/api_utils/user_agent.py +++ b/newapi/api_utils/user_agent.py @@ -16,6 +16,6 @@ def default_user_agent(): # --- li = f"{tool} bot/1.0 (https://{tool}.toolforge.org/; tools.{tool}@toolforge.org)" # --- - # printe.output(f"default_user_agent: {li}") + # logger.info(f"default_user_agent: {li}") # --- return li diff --git a/newapi/api_utils/wd_sparql.py b/newapi/api_utils/wd_sparql.py index f46c452..139ea9d 100644 --- a/newapi/api_utils/wd_sparql.py +++ b/newapi/api_utils/wd_sparql.py @@ -7,10 +7,13 @@ """ +import logging import sys from SPARQLWrapper import JSON, SPARQLWrapper +logger = logging.getLogger(__name__) + def get_query_data(query): """Retrieve query data from the Wikidata SPARQL endpoint. @@ -34,7 +37,10 @@ def get_query_data(query): # endpoint_url = "https://query-main.wikidata.org/sparql" endpoint_url = "https://query.wikidata.org/sparql" # --- - user_agent = "WDQS-example Python/%s.%s" % (sys.version_info[0], sys.version_info[1]) + user_agent = "WDQS-example Python/%s.%s" % ( + sys.version_info[0], + sys.version_info[1], + ) # --- sparql = SPARQLWrapper(endpoint_url, agent=user_agent) # --- @@ -46,9 +52,7 @@ def get_query_data(query): try: data = sparql.query().convert() except Exception as e: - # exception_err(e, text=f"API/tools.py quoteurl: Exception: {e}") - print("API/tools.py get_query_data: Exception: e:") - print(e) + logger.error(f"API/tools.py : Exception: {e}") # --- return data diff --git a/newapi/logging_config.py b/newapi/logging_config.py new file mode 100644 index 0000000..58911d5 --- /dev/null +++ b/newapi/logging_config.py @@ -0,0 +1,160 @@ +""" +Logging configuration with colored output. +""" + +import functools +import logging +import re +import sys + +import colorlog + + +@functools.lru_cache(maxsize=1) +def get_color_table() -> dict[str, str]: + """Build a mapping of color names to ANSI templates.""" + # new Define the color codes for different colors + color_numbers = { + "red": 91, + "green": 92, + "yellow": 93, + "blue": 94, + "purple": 95, + "cyan": 96, + "white": 97, + "black": 98, + "grey": 99, + "gray": 100, + "underline": 4, + "invert": 7, + "blink": 5, + "lightblack": 108, + "bold": 1, + } + data = {x: f"\033[{v}m%s\033[00m" for x, v in color_numbers.items()} + + # Add light versions of the colors to the color table + for color in ["purple", "yellow", "blue", "red", "green", "cyan", "gray"]: + data[f"light{color}"] = data.get(color, 0) + + # Add some additional color names to the color table + data["aqua"] = data.get("cyan", 0) + data["lightaqua"] = data.get("cyan", 0) + data["lightgrey"] = data.get("gray", 0) + data["grey"] = data.get("gray", 0) + data["lightwhite"] = data.get("gray", 0) + data["light"] = 0 + + return data + + +def format_colored_text(textm: str) -> str: + """ + Prints the given text with color formatting. + + The text can contain color tags like '<>' where 'color' is the name of the color. + The color will be applied to the text that follows the tag, until the end of the string or until a '<>' tag is found. + + :param textm: The text to print. Can contain color tags. + """ + color_table = get_color_table() + # Define a pattern for color tags + _color_pat = r"((:?\w+|previous);?(:?\w+|previous)?)" + # Compile a regex for color tags + colorTagR = re.compile(rf"(?:\03{{|<<){_color_pat}(?:}}|>>)") + + # Initialize a stack for color tags + color_stack = ["default"] + + # If the input is not a string, print it as is and return + if not isinstance(textm, str): + return textm + + # If the text does not contain any color tags, print it as is and return + if "\03" not in textm and "<<" not in textm: + return textm + + # Split the text into parts based on the color tags + text_parts = colorTagR.split(textm) + ["default"] + + # Enumerate the parts for processing + enu = enumerate(zip(text_parts[::4], text_parts[1::4], strict=False)) + + # Initialize the string to be printed + toprint = "" + + # Process each part of the text + for _, (text, next_color) in enu: + # Get the current color from the color stack + # print(f"i: {index}, {text=}, {next_color=}") + current_color = color_stack[-1] + + # If the next color is 'previous', pop the color stack to get the previous color + if next_color == "previous": + if len(color_stack) > 1: # keep the last element in the stack + color_stack.pop() + next_color = color_stack[-1] + else: + # If the next color is not 'previous', add it to the color stack + color_stack.append(next_color) + + # Get the color code for the current color + cc = color_table.get(current_color, "") + + # If the color code is not empty, apply it to the text + if cc: + text = cc % text + + # Add the colored text to the string to be printed + toprint += text + + # Print the final colored text + return toprint + + +def wrap_color_messages(format_message): + """Wrap the color messages to include additional context.""" + + def wrapper(record): + # Add custom attributes or modify the record as needed + return format_colored_text(format_message(record)) + + return wrapper + + +def setup_logging( + level: str = "WARNING", + name: str = __name__, +) -> None: + """ + Configure logging for the entire project namespace only. + """ + project_logger = logging.getLogger(name) + + if project_logger.handlers: + return + + numeric_level = getattr(logging, level.upper(), logging.INFO) + project_logger.setLevel(numeric_level) + project_logger.propagate = False + + formatter = colorlog.ColoredFormatter( + fmt="%(filename)s:%(lineno)s %(funcName)s() - %(log_color)s%(levelname)-s %(reset)s%(message)s", + log_colors={ + "DEBUG": "cyan", + "INFO": "green", + "WARNING": "yellow", + "ERROR": "red", + "CRITICAL": "red,bg_white", + }, + ) + + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(formatter) + # message colorizer + formatter.formatMessage = wrap_color_messages(formatter.formatMessage) + + handler.setLevel(numeric_level) + handler.propagate = False + + project_logger.addHandler(handler) diff --git a/newapi/pages_bots/all_apis.py b/newapi/pages_bots/all_apis.py index 4e690dc..66bdfce 100644 --- a/newapi/pages_bots/all_apis.py +++ b/newapi/pages_bots/all_apis.py @@ -9,16 +9,19 @@ new_api = main_api.NEW_API() """ -# --- import functools +import logging from typing import Any -from ..api_utils import printe from ..super.S_API import bot_api from ..super.S_Category import catdepth_new from ..super.S_Page import super_page + +# --- from ..super.super_login import Login +logger = logging.getLogger(__name__) + class ALL_APIS: """ @@ -54,9 +57,7 @@ def _login(self) -> Login: # --- login_bot = Login(self.lang, family=self.family) # --- - printe.output( - f"### <> LoginWrap make new bot for ({self.lang}.{self.family}.org|{self.username})", p=True - ) + logger.info(f"### <> LoginWrap make new bot for ({self.lang}.{self.family}.org|{self.username})") # --- user_tables = { self.family: { diff --git a/newapi/super/S_API/bot.py b/newapi/super/S_API/bot.py index 35cabb6..5ec4ae1 100644 --- a/newapi/super/S_API/bot.py +++ b/newapi/super/S_API/bot.py @@ -4,12 +4,14 @@ """ +import logging import sys -from ...api_utils import printe from ...api_utils.ask_bot import ASK_BOT from ..handel_errors import HANDEL_ERRORS +logger = logging.getLogger(__name__) + yes_answer = ["y", "a", "", "Y", "A", "all", "aaa"] file_name = "bot_api.py" @@ -25,15 +27,15 @@ def __init__(self): def Add_To_Bottom(self, text, summary, title, poss="Head|Bottom"): # --- if not title.strip(): - printe.output('** Add_To_Bottom .. title == ""') + logger.info('** .. title == ""') return False # --- if not text.strip(): - printe.output('** Add_To_Bottom .. text == ""') + logger.info('** .. text == ""') return False # --- - printe.test_print(f"** Add_To_Bottom .. [[{title}]] ") - # printe.showDiff("", text) + logger.debug(f"** .. [[{title}]] ") + # --- user = self.username or getattr(self, "user_login", "") # --- @@ -72,7 +74,7 @@ def Add_To_Bottom(self, text, summary, title, poss="Head|Bottom"): result = data.get("result", "") # --- if result == "Success": - printe.output(f"<>** True. Add_To_Bottom title:({title})") + logger.info(f"<>** True. title:({title})") return True # --- error = results.get("error", {}) @@ -85,9 +87,17 @@ def Add_To_Bottom(self, text, summary, title, poss="Head|Bottom"): # --- return True - def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, return_dict=False): - # --- - printe.output(f"<> def move [[{old_title}]] to [[{to}]] ") + def move( + self, + old_title, + to, + reason="", + noredirect=False, + movesubpages=False, + return_dict=False, + ): + # --- + logger.info(f"<> def [[{old_title}]] to [[{to}]] ") # --- params = { "action": "move", @@ -107,7 +117,7 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r params["reason"] = reason # --- if old_title == to: - printe.test_print(f"<>** old_title == to {to} ") + logger.debug(f"<>** old_title == to {to} ") return {} # --- message = f"Do you want to move page:[[{old_title}]] to [[{to}]]?" @@ -121,7 +131,7 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r # { "move": { "from": "d", "to": "d2", "reason": "wrong", "redirectcreated": true, "moveoverredirect": false } } # --- if not data: - printe.output("no data") + logger.info("no data") return {} # --- _expend_data = { @@ -134,7 +144,11 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r "talkmove-errors": [ { "message": "content-not-allowed-here", - "params": ["Structured Discussions board", "User talk:Mr. Ibrahem/x", "main"], + "params": [ + "Structured Discussions board", + "User talk:Mr. Ibrahem/x", + "main", + ], "code": "contentnotallowedhere", "type": "error", }, @@ -147,12 +161,22 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r ], "subpages": { "errors": [ - {"message": "cant-move-subpages", "params": [], "code": "cant-move-subpages", "type": "error"} + { + "message": "cant-move-subpages", + "params": [], + "code": "cant-move-subpages", + "type": "error", + } ] }, "subpages-talk": { "errors": [ - {"message": "cant-move-subpages", "params": [], "code": "cant-move-subpages", "type": "error"} + { + "message": "cant-move-subpages", + "params": [], + "code": "cant-move-subpages", + "type": "error", + } ] }, } @@ -165,7 +189,7 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r # elif "Please choose another name." in r4: # --- if move_done: - printe.output("<>** true.") + logger.info("<>** true.") # --- if return_dict: return move_done @@ -174,7 +198,7 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r # --- if error: if error_code == "ratelimited": - printe.output("<> move ratelimited:") + logger.info("<> ratelimited:") return self.move( old_title, to, @@ -185,11 +209,11 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r ) if error_code == "articleexists": - printe.output("<> articleexists") + logger.info("<> articleexists") return "articleexists" - printe.output("<> error") - printe.output(error) + logger.info("<> error") + logger.info(error) return {} # --- @@ -242,7 +266,7 @@ def Parse_Text(self, line, title): def upload_by_file(self, file_name, text, file_path, comment="", ignorewarnings=False): # --- - printe.output(f"<> def upload_by_file. {file_name=}") + logger.info(f"<> def . {file_name=}") # --- if file_name.startswith("File:"): file_name = file_name.replace("File:", "") @@ -250,7 +274,7 @@ def upload_by_file(self, file_name, text, file_path, comment="", ignorewarnings= if file_name.startswith("ملف:"): file_name = file_name.replace("ملف:", "") # --- - printe.output(f"<> {file_path=}...") + logger.info(f"<> {file_path=}...") # --- params = { "action": "upload", @@ -274,11 +298,11 @@ def upload_by_file(self, file_name, text, file_path, comment="", ignorewarnings= duplicate = upload_result.get("warnings", {}).get("duplicate", [""])[0].replace("_", " ") # --- if success: - printe.output(f"<> ** upload true .. [[File:{file_name}]] ") + logger.info(f"<> ** upload true .. [[File:{file_name}]] ") return True # --- if duplicate: - printe.output(f"<> ** duplicate file: {duplicate}.") + logger.info(f"<> ** duplicate file: {duplicate}.") # --- return data diff --git a/newapi/super/S_API/bot_api.py b/newapi/super/S_API/bot_api.py index b43478c..09557fa 100644 --- a/newapi/super/S_API/bot_api.py +++ b/newapi/super/S_API/bot_api.py @@ -42,7 +42,7 @@ """ import datetime -import sys +import logging import time from collections.abc import KeysView from datetime import timedelta @@ -50,10 +50,11 @@ # --- import tqdm -from ...api_utils import printe from ...api_utils.lang_codes import change_codes from .bot import BOTS_APIS +logger = logging.getLogger(__name__) + class NEW_API(BOTS_APIS): def __init__(self, login_bot, lang="", family="wikipedia"): @@ -74,17 +75,47 @@ def __init__(self, login_bot, lang="", family="wikipedia"): # --- super().__init__() - def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=None, do_error=False, max_retry=0): + def post_params( + self, + params, + Type="get", + addtoken=False, + GET_CSRF=True, + files=None, + do_error=False, + max_retry=0, + ): # --- return self.login_bot.post_params( - params, Type=Type, addtoken=addtoken, GET_CSRF=GET_CSRF, files=files, do_error=do_error, max_retry=max_retry + params, + Type=Type, + addtoken=addtoken, + GET_CSRF=GET_CSRF, + files=files, + do_error=do_error, + max_retry=max_retry, ) def post_continue( - self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None + self, + params, + action, + _p_="pages", + p_empty=None, + Max=500000, + first=False, + _p_2="", + _p_2_empty=None, ): return self.login_bot.post_continue( - params, action, _p_=_p_, p_empty=p_empty, Max=Max, first=first, _p_2=_p_2, _p_2_empty=_p_2_empty + params, + action, + _p_=_p_, + p_empty=p_empty, + Max=Max, + first=first, + _p_2=_p_2, + _p_2_empty=_p_2_empty, ) def get_username(self): @@ -119,7 +150,7 @@ def Find_pages_exists_or_not(self, liste, get_redirect=False, noprint=False): # --- if not json1: if not noprint: - printe.output("<> error when Find_pages_exists_or_not") + logger.info("<> error when ") # return table continue # --- @@ -162,7 +193,7 @@ def Find_pages_exists_or_not(self, liste, get_redirect=False, noprint=False): exists += 1 # --- if not noprint: - printe.output(f"Find_pages_exists_or_not : missing:{missing}, exists: {exists}, redirects: {redirects}") + logger.info(f"Find_pages_exists_or_not : missing:{missing}, exists: {exists}, redirects: {redirects}") # --- return table @@ -201,7 +232,7 @@ def Find_pages_exists_or_not_with_qids( # --- if not json1: if not noprint: - printe.output("<> error when Find_pages_exists_or_not") + logger.info("<> error when Find_pages_exists_or_not") # return table continue # --- @@ -260,16 +291,24 @@ def Find_pages_exists_or_not_with_qids( exists += 1 # --- if not noprint: - printe.output(f"Find_pages_exists_or_not : missing:{missing}, exists: {exists}, redirects: {redirects}") + logger.info(f"Find_pages_exists_or_not : missing:{missing}, exists: {exists}, redirects: {redirects}") # --- if return_all_jsons: return table, all_jsons # --- return table - def Get_All_pages(self, start="", namespace="0", limit="max", apfilterredir="", ppprop="", limit_all=100000): + def Get_All_pages( + self, + start="", + namespace="0", + limit="max", + apfilterredir="", + ppprop="", + limit_all=100000, + ): # --- - printe.test_print( + logger.debug( f"Get_All_pages for start:{start}, limit:{limit},namespace:{namespace},apfilterredir:{apfilterredir}" ) # --- @@ -298,13 +337,13 @@ def Get_All_pages(self, start="", namespace="0", limit="max", apfilterredir="", # --- newp = self.post_continue(params, "query", _p_="allpages", p_empty=[], Max=limit_all) # --- - printe.test_print(f"<> --- Get_All_pages : find {len(newp)} pages.") + logger.debug(f"<> --- : find {len(newp)} pages.") # --- Main_table = [x["title"] for x in newp] # --- - printe.test_print(f"len of Main_table {len(Main_table)}.") + logger.debug(f"len of Main_table {len(Main_table)}.") # --- - printe.output(f"bot_api.py Get_All_pages : find {len(Main_table)} pages.") + logger.info(f"bot_api.py : find {len(Main_table)} pages.") # --- return Main_table @@ -329,7 +368,7 @@ def PrefixSearch(self, pssearch="", ns="0", pslimit="max", limit_all=100000): list: A list of titles that match the prefix search. """ # --- - printe.test_print(f"PrefixSearch for start:{pssearch}, pslimit:{pslimit}, ns:{ns}") + logger.debug(f" for start:{pssearch}, pslimit:{pslimit}, ns:{ns}") # --- pssearch = pssearch.strip() if pssearch else "" # --- @@ -357,21 +396,27 @@ def PrefixSearch(self, pssearch="", ns="0", pslimit="max", limit_all=100000): # --- newp = self.post_continue(params, "query", _p_="prefixsearch", p_empty=[], Max=limit_all) # --- - printe.test_print(f"<> --- PrefixSearch : find {len(newp)} pages.") + logger.debug(f"<> --- : find {len(newp)} pages.") # --- Main_table = [x["title"] for x in newp] # --- - printe.test_print(f"len of Main_table {len(Main_table)}.") + logger.debug(f"len of Main_table {len(Main_table)}.") # --- - printe.output(f"bot_api.py PrefixSearch : find {len(Main_table)} pages.") + logger.info(f"bot_api.py : find {len(Main_table)} pages.") # --- return Main_table def Get_All_pages_generator( - self, start="", namespace="0", limit="max", filterredir="", ppprop="", limit_all=100000 + self, + start="", + namespace="0", + limit="max", + filterredir="", + ppprop="", + limit_all=100000, ): # --- - printe.test_print( + logger.debug( f"Get_All_pages_generator for start:{start}, limit:{limit},namespace:{namespace},filterredir:{filterredir}" ) # --- @@ -401,19 +446,27 @@ def Get_All_pages_generator( # --- newp = self.post_continue(params, "query", _p_="pages", p_empty=[], Max=limit_all) # --- - printe.test_print(f"<> --- Get_All_pages_generator : find {len(newp)} pages.") + logger.debug(f"<> --- Get_All_pages_generator : find {len(newp)} pages.") # --- Main_table = {x["title"]: x for x in newp} # --- - printe.test_print(f"len of Main_table {len(Main_table)}.") + logger.debug(f"len of Main_table {len(Main_table)}.") # --- - printe.output(f"bot_api.py Get_All_pages_generator : find {len(Main_table)} pages.") + logger.info(f"bot_api.py Get_All_pages_generator : find {len(Main_table)} pages.") # --- return Main_table - def Search(self, value="", ns="*", offset="", srlimit="max", RETURN_dict=False, addparams=None): + def Search( + self, + value="", + ns="*", + offset="", + srlimit="max", + RETURN_dict=False, + addparams=None, + ): # --- - printe.test_print(f'bot_api.Search for "{value}",ns:{ns}') + logger.debug(f'bot_api. for "{value}",ns:{ns}') # --- if not srlimit: srlimit = "max" @@ -448,7 +501,7 @@ def Search(self, value="", ns="*", offset="", srlimit="max", RETURN_dict=False, else: results.append(pag["title"]) # --- - printe.test_print(f'bot_api.Search find "{len(search)}" all result: {len(results)}') + logger.debug(f'bot_api. find "{len(search)}" all result: {len(results)}') # --- return results @@ -496,7 +549,7 @@ def Get_Newpages( Main_table = [x["title"] for x in json1] - printe.test_print(f'bot_api.Get_Newpages find "{len(Main_table)}" result. s') + logger.debug(f'bot_api. find "{len(Main_table)}" result. s') return Main_table @@ -564,7 +617,7 @@ def Get_langlinks_for_list(self, titles, targtsitecode="", numbes=40): """ # --- - printe.test_print(f'bot_api.Get_langlinks_for_list for "{len(titles)} pages". in wiki:{self.lang}') + logger.debug(f'bot_api.Get_langlinks_for_list for "{len(titles)} pages". in wiki:{self.lang}') # --- if targtsitecode.endswith("wiki"): targtsitecode = targtsitecode[:-4] @@ -587,7 +640,7 @@ def Get_langlinks_for_list(self, titles, targtsitecode="", numbes=40): # --- if targtsitecode: params["lllang"] = targtsitecode - printe.test_print(f'params["lllang"] = {targtsitecode}') + logger.debug(f'params["lllang"] = {targtsitecode}') # --- find_targtsitecode = 0 normalized = {} @@ -596,12 +649,12 @@ def Get_langlinks_for_list(self, titles, targtsitecode="", numbes=40): for title_chunk in self.chunk_titles(titles, chunk_size=numbes): params["titles"] = "|".join(title_chunk) # --- - # printe.test_print(f'bot_api.Get_langlinks_for_list work for {len(group)} pages') + # logger.debug(f'bot_api.Get_langlinks_for_list work for {len(group)} pages') # --- json1 = self.post_params(params) # --- if not json1: - printe.output("bot_api.Get_langlinks_for_list json1 is empty") + logger.info("bot_api. json1 is empty") continue # --- _error = json1.get("error", {}) @@ -629,7 +682,7 @@ def Get_langlinks_for_list(self, titles, targtsitecode="", numbes=40): if lang["lang"] == targtsitecode: find_targtsitecode += 1 # --- - printe.output( + logger.info( f'bot_api.Get_langlinks_for_list find "{len(table)}" in table,find_targtsitecode:{targtsitecode}:{find_targtsitecode}' ) # --- @@ -686,7 +739,15 @@ def get_pageassessments(self, titles): "formatversion": 2, } # --- - results = self.post_continue(params, "query", "pages", [], first=False, _p_2="pageassessments", _p_2_empty=[]) + results = self.post_continue( + params, + "query", + "pages", + [], + first=False, + _p_2="pageassessments", + _p_2_empty=[], + ) # --- return results @@ -771,17 +832,17 @@ def querypage_list(self, qppage="Wantedcategories", qplimit=None, Max=None): ] # --- if qppage not in qppage_values: - printe.output(f"<> qppage {qppage} not in qppage_values.") + logger.info(f"<> qppage {qppage} not in qppage_values.") # --- results = self.post_continue(params, "query", _p_="querypage", p_empty=[], Max=Max) # --- - printe.test_print(f"querypage_list len(results) = {len(results)}") + logger.debug(f" len(results) = {len(results)}") # --- return results def Get_template_pages(self, title, namespace="*", Max=10000): # --- - printe.test_print(f'Get_template_pages for template:"{title}", limit:"{Max}",namespace:"{namespace}"') + logger.debug(f'Get_template_pages for template:"{title}", limit:"{Max}",namespace:"{namespace}"') # --- params = { "action": "query", @@ -798,7 +859,7 @@ def Get_template_pages(self, title, namespace="*", Max=10000): # { "pageid": 2973452, "ns": 100, "title": "بوابة:سباق الدراجات الهوائية" } pages = [x["title"] for x in results] # --- - printe.output(f"mdwiki_api.py Get_template_pages : find {len(pages)} pages.") + logger.info(f"mdwiki_api.py : find {len(pages)} pages.") # --- return pages @@ -807,7 +868,7 @@ def Get_image_url(self, title): if not title.startswith("File:") and not title.startswith("ملف:"): title = f"File:{title}" # --- - printe.test_print(f'Get_image_url for file:"{title}":') + logger.debug(f' for file:"{title}":') # --- params = { "action": "query", @@ -830,7 +891,7 @@ def Get_image_url(self, title): # --- url = data.get("imageinfo", [{}])[0].get("url", "") # --- - printe.output(f"Get_image_url: image url: {url}") + logger.info(f": image url: {url}") # --- return url @@ -839,7 +900,7 @@ def Get_imageinfo(self, title): if not title.startswith("File:") and not title.startswith("ملف:"): title = f"File:{title}" # --- - printe.test_print(f'Get_imageinfo for file:"{title}":') + logger.debug(f' for file:"{title}":') # --- params = { "action": "query", @@ -880,7 +941,7 @@ def pageswithprop(self, pwppropname="unlinkedwikibase_id", pwplimit=None, Max=No # --- results = self.post_continue(params, "query", _p_="pageswithprop", p_empty=[], Max=Max) # --- - printe.test_print(f"pageswithprop len(results) = {len(results)}") + logger.debug(f" len(results) = {len(results)}") # --- return results @@ -950,7 +1011,7 @@ def users_infos(self, ususers=[]): "ususers": "Mr.Ibrahembot", } # --- - all_usprops = [ + _all_usprops = [ "groups", "implicitgroups", "cancreate", @@ -970,7 +1031,7 @@ def users_infos(self, ususers=[]): # --- results = self.post_continue(params, "query", _p_="users", p_empty=[]) # --- - printe.test_print(f"users_infos len(results) = {len(results)}") + logger.debug(f" len(results) = {len(results)}") # --- results = [dict(x) for x in results] # --- diff --git a/newapi/super/S_Category/bot.py b/newapi/super/S_Category/bot.py index b31ac9c..5b54aa6 100644 --- a/newapi/super/S_Category/bot.py +++ b/newapi/super/S_Category/bot.py @@ -5,10 +5,11 @@ """ import copy +import logging import tqdm -from ...api_utils import printe +logger = logging.getLogger(__name__) ns_list = { "0": "", @@ -66,10 +67,25 @@ def __init__(self, login_bot, title, **kwargs): # --- self.prase_params(**kwargs) - def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=None, do_error=False, max_retry=0): + def post_params( + self, + params, + Type="get", + addtoken=False, + GET_CSRF=True, + files=None, + do_error=False, + max_retry=0, + ): # --- return self.login_bot.post_params( - params, Type=Type, addtoken=addtoken, GET_CSRF=GET_CSRF, files=files, do_error=do_error, max_retry=max_retry + params, + Type=Type, + addtoken=addtoken, + GET_CSRF=GET_CSRF, + files=files, + do_error=do_error, + max_retry=max_retry, ) def get_revids(self): @@ -277,7 +293,7 @@ def get_cat_new(self, cac): d += 1 # --- if self.limit > 0 and len(results) >= self.limit: - printe.output(f"<> limit:{self.limit} reached, len of results: {len(results)} break ..") + logger.info(f"<> limit:{self.limit} reached, len of results: {len(results)} break ..") break # --- if continue_params: @@ -343,14 +359,12 @@ def subcatquery_(self, **kwargs): new_tab2 = [] # --- if self.limit > 0 and len(self.result_table) >= self.limit: - printe.output( - f"<> limit:{self.limit} reached, len of results: {len(self.result_table)} break .." - ) + logger.info(f"<> limit:{self.limit} reached, len of results: {len(self.result_table)} break ..") break # --- depth_done += 1 # --- - # printe.output(f"<> work in subcats: {len(new_list)}, depth:{depth_done}/{self.depth}:") + # logger.info(f"<> work in subcats: {len(new_list)}, depth:{depth_done}/{self.depth}:") # --- for cat in tqdm.tqdm(new_list): # --- @@ -367,7 +381,11 @@ def subcatquery_(self, **kwargs): # --- # sort self.result_table by timestamp if not self.no_gcmsort: - soro = sorted(self.result_table.items(), key=lambda item: self.timestamps.get(item[0], 0), reverse=True) + soro = sorted( + self.result_table.items(), + key=lambda item: self.timestamps.get(item[0], 0), + reverse=True, + ) self.result_table = {k: v for k, v in soro} # --- return self.result_table diff --git a/newapi/super/S_Category/catdepth_new.py b/newapi/super/S_Category/catdepth_new.py index 82ad3b0..37553cb 100644 --- a/newapi/super/S_Category/catdepth_new.py +++ b/newapi/super/S_Category/catdepth_new.py @@ -1,12 +1,19 @@ -""" """ +""" +from newapi.ncc_page import CatDepth +# cat_members = CatDepth(title, sitecode='www', family="nccommons", depth=0, ns=10, nslist=[], onlyns=False, tempyes=[]) + +""" + +import logging import sys import time from functools import lru_cache -from ...api_utils import printe from .bot import CategoryDepth +logger = logging.getLogger(__name__) + SITECODE = "en" FAMILY = "wikipedia" @@ -57,7 +64,7 @@ def subcatquery(login_bot, title, sitecode=SITECODE, family=FAMILY, **kwargs): args2 = args_group(title, kwargs) # --- if print_s: - printe.output( + logger.info( f"<> catdepth_new.py sub cat query for {sitecode}:{title}, depth:{args2['depth']}, ns:{args2['ns']}, onlyns:{args2['onlyns']}" ) # --- @@ -73,12 +80,12 @@ def subcatquery(login_bot, title, sitecode=SITECODE, family=FAMILY, **kwargs): delta = int(time.time() - start) # --- if "printresult" in sys.argv: - printe.output(result) + logger.info(result) # --- if print_s: lenpages = bot.get_len_pages() # --- - printe.output( + logger.info( f"<>catdepth_new.py: find {len(result)} pages({args2['ns']}) in {sitecode}:{title}, depth:{args2['depth']} in {delta} seconds | {lenpages=}" ) # --- diff --git a/newapi/super/S_Page/bot.py b/newapi/super/S_Page/bot.py index 9607088..46b5b7f 100644 --- a/newapi/super/S_Page/bot.py +++ b/newapi/super/S_Page/bot.py @@ -19,8 +19,23 @@ def __init__(self, login_bot): super().__init__() def post_continue( - self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None + self, + params, + action, + _p_="pages", + p_empty=None, + Max=500000, + first=False, + _p_2="", + _p_2_empty=None, ): return self.login_bot.post_continue( - params, action, _p_=_p_, p_empty=p_empty, Max=Max, first=first, _p_2=_p_2, _p_2_empty=_p_2_empty + params, + action, + _p_=_p_, + p_empty=p_empty, + Max=Max, + first=first, + _p_2=_p_2, + _p_2_empty=_p_2_empty, ) diff --git a/newapi/super/S_Page/super_page.py b/newapi/super/S_Page/super_page.py index 8d7e360..d8e1329 100644 --- a/newapi/super/S_Page/super_page.py +++ b/newapi/super/S_Page/super_page.py @@ -44,20 +44,18 @@ """ -import os +import logging import sys -from warnings import warn - import wikitextparser as wtp -from ...api_utils import botEdit, printe, txtlib +from ...api_utils import botEdit, txtlib from ...api_utils.ask_bot import ASK_BOT -from ...api_utils.except_err import exception_err, warn_err from ...api_utils.lang_codes import change_codes from .ar_err import find_edit_error from .bot import PAGE_APIS from .data import CategoriesData, Content, LinksData, Meta, RevisionsData, TemplateData +logger = logging.getLogger(__name__) print_test = {1: "test" in sys.argv} @@ -124,7 +122,7 @@ def false_edit(self): if len(self.newtext) < 0.1 * len(self.text): text_err = f"Edit will remove 90% of the text. {len(self.newtext)} < 0.1 * {len(self.text)}" text_err += f"title: {self.title}, summary: {self.content.summary}" - exception_err("", text=text_err) + logger.exception("", text=text_err) return True # --- if self.lang == "ar" and self.ns == 0: @@ -156,7 +154,7 @@ def import_page(self, family="wikipedia"): # --- done = data.get("import", [{}])[0].get("revisions", 0) # --- - printe.output(f"<> imported {done} revisions") + logger.info(f"<> imported {done} revisions") # --- return data @@ -231,7 +229,7 @@ def get_text(self, redirects=False): for k, v in pages.items(): # --- if print_test[1] or "printdata" in sys.argv: - warn(warn_err(f"v:{str(v)}"), UserWarning) + logger.warning(f"v:{str(v)}") # --- if "ns" in v: self.ns = v["ns"] # ns = 0 ! @@ -401,7 +399,7 @@ def get_redirect_target(self): to = redirects.get("to", "") # --- if to: - printe.output(f"<>Page:({self.title}) redirect to ({to})") + logger.info(f"<>Page:({self.title}) redirect to ({to})") # --- return to @@ -505,7 +503,7 @@ def isDisambiguation(self): self.meta.is_Disambig = self.title.endswith("(توضيح)") or self.title.endswith("(disambiguation)") # --- if self.meta.is_Disambig: - printe.output(f'<> page "{self.title}" is Disambiguation / توضيح') + logger.info(f'<> page "{self.title}" is Disambiguation / توضيح') # --- return self.meta.is_Disambig @@ -555,7 +553,7 @@ def get_wiki_links_from_text(self): parsed = wtp.parse(self.text) wikilinks = parsed.wikilinks # --- - # printe.output(f'wikilinks:{str(wikilinks)}') + # logger.info(f'wikilinks:{str(wikilinks)}') # --- # for x in wikilinks: # print(x.title) @@ -571,7 +569,7 @@ def Get_tags(self, tag=""): parsed = wtp.parse(self.text) tags = parsed.get_tags() # --- - # printe.output(f'tags:{str(tags)}') + # logger.info(f'tags:{str(tags)}') # --- if not tag: return tags @@ -653,7 +651,7 @@ def exists(self): if not self.meta.Exists: self.get_text() if not self.meta.Exists: - printe.output(f'page "{self.title}" not exists in {self.lang}:{self.family}') + logger.info(f'page "{self.title}" not exists in {self.lang}:{self.family}') return self.meta.Exists def namespace(self): @@ -750,8 +748,8 @@ def save(self, newtext="", summary="", nocreate=1, minor="0", tags="", nodiff=Fa if result.lower() == "success": self.text = newtext self.user = "" - printe.output(f"<> ** true .. [[{self.lang}:{self.family}:{self.title}]] ") - # printe.output('Done True...') + logger.info(f"<> ** true .. [[{self.lang}:{self.family}:{self.title}]] ") + # logger.info('Done True...') # --- if "printpop" in sys.argv: print(pop) @@ -784,7 +782,7 @@ def purge(self): data = self.post_params(params, addtoken=True) # --- if not data: - printe.output("<> ** purge error. ") + logger.info("<> ** purge error. ") return False # --- title2 = self.title @@ -792,7 +790,7 @@ def purge(self): # 'normalized': [{'from': 'وب:ملعب', 'to': 'ويكيبيديا:ملعب'}]} # --- for x in data.get("normalized", []): - # printe.output(f"normalized from {x['from']} to {x['to']}") + # logger.info(f"normalized from {x['from']} to {x['to']}") if x["from"] == self.title: title2 = x["to"] break @@ -803,7 +801,7 @@ def purge(self): if title2 == ti and "purged" in t: return True if "missing" in t: - printe.output(f"page \"{t['title']}\" missing") + logger.info(f"page \"{t['title']}\" missing") return "missing" return False @@ -865,8 +863,8 @@ def Create(self, text="", summary="", nodiff="", noask=False): # --- self.text = text # --- - printe.output(f"<> ** true .. [[{self.lang}:{self.family}:{self.title}]] ") - # printe.output('Done True... time.sleep() ') + logger.info(f"<> ** true .. [[{self.lang}:{self.family}:{self.title}]] ") + # logger.info('Done True... time.sleep() ') # --- self.revisions_data.pageid = edit.get("pageid") or self.revisions_data.pageid self.revisions_data.revid = edit.get("newrevid") or self.revisions_data.revid @@ -915,7 +913,17 @@ def page_backlinks(self, ns=0): # --- return self.links_data.back_links - def page_links(self): + def page_links(self) -> list: + """ + Get the links on the page. + Return: + list: A list of links on the page, where each link is represented as a dictionary containing its namespace, title, and existence status. + Example of returned data: + [ + {'ns': 14, 'title': 'تصنيف:مقالات بحاجة لشريط بوابات', 'exists': True}, + {'ns': 14, 'title': 'تصنيف:مقالات بحاجة لصندوق معلومات', 'exists': False} + ] + """ params = { "action": "parse", "prop": "links", @@ -925,7 +933,7 @@ def page_links(self): # data = self.post_params(params) # data = data.get('parse', {}).get('links', []) # --- - data = self.post_continue(params, "parse", _p_="links", p_empty=[]) + data: list = self.post_continue(params, "parse", _p_="links", p_empty=[]) # --- # [{'ns': 14, 'title': 'تصنيف:مقالات بحاجة لشريط بوابات', 'exists': True}, {'ns': 14, 'title': 'تصنيف:مقالات بحاجة لصندوق معلومات', 'exists': False}] # --- diff --git a/newapi/super/bot.py b/newapi/super/bot.py index fc3b8dd..524b58a 100644 --- a/newapi/super/bot.py +++ b/newapi/super/bot.py @@ -7,15 +7,13 @@ """ -import os import logging +import os import sys from http.cookiejar import MozillaCookieJar import requests -from ..api_utils import printe -from ..api_utils.except_err import exception_err from ..api_utils.user_agent import default_user_agent from .cookies_bot import del_cookies_file, get_file_name from .params_help import PARAMS_HELPS @@ -58,7 +56,9 @@ def __init__(self) -> None: super().__init__() def log_error(self, result, action, params=None) -> None: - logger.error(f"Error occurred: {result}, Action: {action}, Params: {params}") + good_result = [200, "success"] + if result not in good_result: + logger.error(f"Error occurred: {result}, Action: {action}, Params: {params}") def add_User_tables(self, family, table, lang="") -> None: # --- @@ -110,10 +110,8 @@ def log_in(self) -> bool: Bot_passwords = self.password.find("@") != -1 logins_count[1] += 1 - printe.output(f"<<{color}>> {botname}/page.py: Log_to_wiki {self.endpoint} count:{logins_count[1]}") - printe.output( - f"{botname}/page.py: log to {self.lang}.{self.family}.org user:{self.username}, ({Bot_passwords=})" - ) + logger.info(f"<<{color}>> {botname}/page.py: Log_to_wiki {self.endpoint} count:{logins_count[1]}") + logger.info(f"{botname}/page.py: log to {self.lang}.{self.family}.org user:{self.username}, ({Bot_passwords=})") logintoken = self.get_logintoken() @@ -123,7 +121,7 @@ def log_in(self) -> bool: success = self.get_login_result(logintoken) if success: - printe.output("<> new_api login Success") + logger.info("<> new_api login Success") return True else: return False @@ -144,12 +142,10 @@ def get_logintoken(self) -> str: self.log_error(r11.status_code, "logintoken") # --- if not str(r11.status_code).startswith("2"): - printe.output( - f"<> {botname} {r11.status_code} Server Error: Server Hangup for url: {self.endpoint}" - ) + logger.info(f"<> {botname} {r11.status_code} Server Error: Server Hangup for url: {self.endpoint}") # --- except Exception as e: - exception_err(e) + logger.exception(e) return "" jsson1 = {} @@ -158,14 +154,14 @@ def get_logintoken(self) -> str: jsson1 = r11.json() except Exception as e: print(r11.text) - exception_err(e) + logger.exception(e) return "" return jsson1.get("query", {}).get("tokens", {}).get("logintoken") or "" def get_login_result(self, logintoken) -> bool: if not self.password: - printe.output("No password") + logger.info("No password") return False r2_params = { @@ -181,7 +177,7 @@ def get_login_result(self, logintoken) -> bool: try: req = seasons_by_lang[self.sea_key].request("POST", self.endpoint, data=r2_params, headers=self.headers) except Exception as e: - exception_err(e) + logger.exception(e) return False # --- r22 = {} @@ -190,7 +186,7 @@ def get_login_result(self, logintoken) -> bool: try: r22 = req.json() except Exception as e: - exception_err(e) + logger.exception(e) print(req.text) return False # --- @@ -206,10 +202,10 @@ def get_login_result(self, logintoken) -> bool: # --- reason = r22.get("login", {}).get("reason", "") # --- - # exception_err(r22) + # logger.exception(r22) # --- if reason == "Incorrect username or password entered. Please try again.": - printe.output(f"user:{self.username}, pass:******") + logger.info(f"user:{self.username}, pass:******") # --- return False @@ -229,7 +225,7 @@ def loged_in(self) -> bool: try: req = seasons_by_lang[self.sea_key].request("POST", self.endpoint, data=params, headers=self.headers) except Exception as e: - exception_err(e) + logger.exception(e) self.log_error("failed", "userinfo") return False # --- @@ -238,7 +234,7 @@ def loged_in(self) -> bool: try: json1 = req.json() except Exception as e: - exception_err(e) + logger.exception(e) print(req.text) return False # --- @@ -260,7 +256,7 @@ def loged_in(self) -> bool: def make_new_session(self) -> None: # --- - printe.output(f"make_new_session:({self.lang}, {self.family}, {self.username})") + logger.info(f":({self.lang}, {self.family}, {self.username})") # --- seasons_by_lang[self.sea_key] = requests.Session() # --- @@ -284,7 +280,7 @@ def make_new_session(self) -> None: if len(self.cookie_jar) > 0: if self.loged_in(): loged_t = True - printe.output(f"<>Cookie Already logged in with user:{self.username_in}") + logger.info(f"<>Cookie Already logged in with user:{self.username_in}") else: loged_t = self.log_in() # --- @@ -297,7 +293,7 @@ def _handle_server_error(self, req0, action, params=None): self.log_error(req0.status_code, action, params=params) # --- if not str(req0.status_code).startswith("2"): - printe.output( + logger.info( f"<> {botname} {req0.status_code} Server Error: Server Hangup for url: {self.endpoint}" ) @@ -306,7 +302,7 @@ def raw_request(self, params, files=None, timeout=30): # TODO: ('toomanyvalues', 'Too many values supplied for parameter "titles". The limit is 50.', 'See https://en.wikipedia.org/w/api.php for API usage. Subscribe to the mediawiki-api-announce mailing list at <https://lists.wikimedia.org/postorius/lists/mediawiki-api-announce.lists.wikimedia.org/> for notice of API deprecations and breaking changes.') # --- if not self.user_table_done: - printe.output("<> user_table_done == False!") + logger.info("<> user_table_done == False!") # do error if "raise" in sys.argv: raise Exception("user_table_done == False!") @@ -324,9 +320,9 @@ def raw_request(self, params, files=None, timeout=30): u_action = params.get("action", "") # --- if "dopost" in sys.argv: - printe.output("<> dopost:::") - printe.output(params) - printe.output("<> :::dopost") + logger.info("<> dopost:::") + logger.info(params) + logger.info("<> :::dopost") req0 = seasons_by_lang[self.sea_key].request("POST", self.endpoint, **args) # --- self._handle_server_error(req0, u_action, params=params) @@ -340,11 +336,11 @@ def raw_request(self, params, files=None, timeout=30): except requests.exceptions.ReadTimeout: self.log_error("ReadTimeout", u_action, params=params) - printe.output(f"<> ReadTimeout: {self.endpoint=}, {timeout=}") + logger.error(f"<> ReadTimeout: {self.endpoint=}, {timeout=}") except Exception as e: self.log_error("Exception", u_action, params=params) - exception_err(e) + logger.exception(e) # --- self._handle_server_error(req0, u_action, params=params) # --- @@ -361,17 +357,17 @@ def post_it(self, params, files=None, timeout=30): self.make_new_session() # --- if not self.username_in: - printe.output("<> no username_in.. action:" + params.get("action")) + logger.info("<> no username_in.. action:" + params.get("action")) # return {} # --- req0 = self.raw_request(params, files=files, timeout=timeout) # --- if not req0: - printe.output("<> no req0.. ") + logger.info("<> no req0.. ") return req0 # --- if req0.headers and req0.headers.get("x-database-lag"): - printe.output("<> x-database-lag.. ") + logger.info("<> x-database-lag.. ") print(req0.headers) # raise # --- @@ -417,12 +413,12 @@ def get_rest_result(self, url) -> dict: req0 = seasons_by_lang[self.sea_key].request("GET", url, headers=self.headers) # --- if not str(req0.status_code).startswith("2"): - printe.output( + logger.info( f"<> {botname} {req0.status_code} Server Error: Server Hangup for url: {self.endpoint}" ) # --- except Exception as e: - exception_err(e) + logger.exception(e) return {} # --- result = {} @@ -431,7 +427,7 @@ def get_rest_result(self, url) -> dict: result = req0.json() except Exception as e: print(req0.text) - exception_err(e) + logger.exception(e) return {} # --- return result diff --git a/newapi/super/bot_new.py b/newapi/super/bot_new.py index 94d7034..40e51b9 100644 --- a/newapi/super/bot_new.py +++ b/newapi/super/bot_new.py @@ -14,8 +14,6 @@ import requests -from ..api_utils import printe -from ..api_utils.except_err import exception_err from ..api_utils.user_agent import default_user_agent from .cookies_bot import del_cookies_file, get_file_name from .mwclient.client import Site @@ -47,7 +45,9 @@ def __init__(self, lang, family): # self._start_() def log_error(self, result, action, params=None) -> None: - logger.error(f"Error occurred: {result}, Action: {action}, Params: {params}") + good_result = [200, "success"] + if result not in good_result: + logger.error(f"Error occurred: {result}, Action: {action}, Params: {params}") def _start_(self, username, password): self.username = username @@ -67,43 +67,48 @@ def __initialize_connection(self): self.connection.headers["User-Agent"] = default_user_agent() # --- if os.path.exists(cookies_file) and self.family != "mdwiki": - # printe.output("<>loading cookies") + # logger.info("<>loading cookies") try: # Load cookies from file, including session cookies self.jar_cookie.load(ignore_discard=True, ignore_expires=True) self.connection.cookies = self.jar_cookie # Tell Requests session to use the cookiejar. except Exception as e: - printe.output("Could not load cookies: %s" % e) + logger.info("Could not load cookies: %s" % e) def __initialize_site(self): self.domain = f"{self.lang}.{self.family}.org" if "dopost" in sys.argv: self.site_mwclient = Site( - self.domain, clients_useragent=self.user_agent, pool=self.connection, force_login=self.force_login + self.domain, + clients_useragent=self.user_agent, + pool=self.connection, + force_login=self.force_login, ) else: try: self.site_mwclient = Site( - self.domain, clients_useragent=self.user_agent, pool=self.connection, force_login=self.force_login + self.domain, + clients_useragent=self.user_agent, + pool=self.connection, + force_login=self.force_login, ) except Exception as e: - printe.output(f"Could not connect to ({self.domain}): %s" % e) + logger.info(f"Could not connect to ({self.domain}): %s" % e) return False def do_login(self): - if not self.force_login: - printe.output("<> do_login(): not self.force_login ") + logger.info("<> (): not self.force_login ") return if not self.site_mwclient: - printe.output(f"no self.ssite_mwclient to ({self.domain})") + logger.info(f"no self.ssite_mwclient to ({self.domain})") return if not self.site_mwclient.logged_in: logins_count[1] += 1 - printe.output(f"<>logging in to ({self.domain}) count:{logins_count[1]}, user: {self.username}") + logger.info(f"<>logging in to ({self.domain}) count:{logins_count[1]}, user: {self.username}") # --- try: login_result = self.site_mwclient.login(username=self.username, password=self.password) @@ -112,10 +117,10 @@ def do_login(self): self.login_done = True except Exception as e: - printe.output(f"Could not login to ({self.domain}): %s" % e) + logger.info(f"Could not login to ({self.domain}): %s" % e) if self.site_mwclient.logged_in: - printe.output(f"<>logged in as {self.site_mwclient.username} to ({self.domain})") + logger.info(f"<>logged in as {self.site_mwclient.username} to ({self.domain})") # Save cookies to file, including session cookies if self.jar_cookie: @@ -133,7 +138,7 @@ def do_request(self, params=None, method="POST"): del params["action"] # --- if not self.site_mwclient: - printe.output(f"no self.ssite_mwclient to ({self.domain})") + logger.info(f"no self.ssite_mwclient to ({self.domain})") self.__initialize_site() self.do_login() # --- @@ -155,7 +160,7 @@ def do_request(self, params=None, method="POST"): if "text" in params: params["text"] = params["text"][:100] # --- - exception_err(e, text=params) + logger.exception(e, text=params) # --- return {} @@ -209,7 +214,7 @@ def make_new_r3_token(self) -> str: try: csrftoken = self.site_mwclient.get_token("csrf") except Exception as e: - printe.output("Could not get token: %s" % e) + logger.info("Could not get token: %s" % e) return False # --- return csrftoken @@ -221,7 +226,7 @@ def log_to_wiki_1(self, do=False) -> str: def raw_request(self, params, files=None, timeout=30): # --- if not self.user_table_done: - printe.output("<> user_table_done == False!") + logger.info("<> user_table_done == False!") # do error if "raise" in sys.argv: raise Exception("user_table_done == False!") @@ -276,7 +281,7 @@ def get_rest_result(self, url) -> dict: result = req0.json() except Exception as e: - exception_err(e) + logger.exception(e) # --- return result diff --git a/newapi/super/cookies_bot.py b/newapi/super/cookies_bot.py index 1467597..97b4669 100644 --- a/newapi/super/cookies_bot.py +++ b/newapi/super/cookies_bot.py @@ -5,6 +5,7 @@ """ +import logging import os import stat import sys @@ -12,7 +13,7 @@ from functools import lru_cache from pathlib import Path -from ..api_utils import printe +logger = logging.getLogger(__name__) statgroup = stat.S_IRWXU | stat.S_IRWXG tool = os.getenv("HOME") @@ -26,9 +27,9 @@ # --- if not ta_dir.exists(): ta_dir.mkdir() - printe.output("<> mkdir:") - printe.output(f"ta_dir:{ta_dir}") - printe.output("<> mkdir:") + logger.info("<> mkdir:") + logger.info(f"ta_dir:{ta_dir}") + logger.info("<> mkdir:") os.chmod(ta_dir, statgroup) @@ -39,9 +40,9 @@ def del_cookies_file(file_path): if file.exists(): try: file.unlink(missing_ok=True) - printe.output(f"<> unlink: file:{file}") + logger.info(f"<> unlink: file:{file}") except Exception as e: - printe.output(f"<> unlink: Exception:{e}") + logger.error(f"<> unlink: Exception:{e}") def get_file_name(lang, family, username) -> Path: @@ -105,7 +106,7 @@ def get_cookies(lang, family, username): cookies = from_folder(lang, family, username) # --- if not cookies: - printe.output(f" <> get_cookies: <> [[{lang}:{family}]] user:{username} <> not found") + logger.info(f" <> get_cookies: <> [[{lang}:{family}]] user:{username} <> not found") return "make_new" # --- return cookies diff --git a/newapi/super/handel_errors.py b/newapi/super/handel_errors.py index f060eb8..25f050c 100644 --- a/newapi/super/handel_errors.py +++ b/newapi/super/handel_errors.py @@ -3,18 +3,24 @@ """ +import logging import sys -# from newapi import printe -from ..api_utils import printe +logger = logging.getLogger(__name__) class HANDEL_ERRORS: def __init__(self): - # printe.output("class HANDEL_ERRORS:") + # logger.info("class HANDEL_ERRORS:") pass - def handel_err(self, error: dict, function: str = "", params: dict = None, do_error: bool = True): + def handel_err( + self, + error: dict, + function: str = "", + params: dict = None, + do_error: bool = True, + ): """Handle errors based on the provided error dictionary. This function processes an error dictionary and performs actions based @@ -56,7 +62,7 @@ def handel_err(self, error: dict, function: str = "", params: dict = None, do_er # --- abusefilter = error.get("abusefilter", "") description = abusefilter.get("description", "") - printe.output(f"<> ** abusefilter-disallowed: {description} ") + logger.info(f"<> ** abusefilter-disallowed: {description} ") if description in [ "تأخير البوتات 3 ساعات", "تأخير البوتات 3 ساعات- 3 من 3", @@ -67,26 +73,26 @@ def handel_err(self, error: dict, function: str = "", params: dict = None, do_er return description # --- if err_code == "no-such-entity": - printe.output("<> ** no-such-entity. ") + logger.info("<> ** no-such-entity. ") return False # --- if err_code == "protectedpage": - printe.output("<> ** protectedpage. ") + logger.info("<> ** protectedpage. ") # return "protectedpage" return False # --- if err_code == "articleexists": - printe.output("<> ** article already created. ") + logger.info("<> ** article already created. ") return "articleexists" # --- if err_code == "maxlag": - printe.output("<> ** maxlag. ") + logger.info("<> ** maxlag. ") return False # --- if do_error: params["data"] = {} params["text"] = {} - printe.error(f"<>{function} ERROR: <>info: {err_info}, {params=}") + logger.error(f"<>{function} ERROR: <>info: {err_info}, {params=}") # --- if "raise" in sys.argv: raise Exception(error) diff --git a/newapi/super/login_wrap.py b/newapi/super/login_wrap.py index c93a891..3fbc3f7 100644 --- a/newapi/super/login_wrap.py +++ b/newapi/super/login_wrap.py @@ -9,9 +9,12 @@ """ -from ..api_utils import printe +import logging + from .super_login import Login +logger = logging.getLogger(__name__) + hases = {} @@ -32,14 +35,13 @@ def LoginWrap(sitecode, family, bots_login_cache, User_tables): hases[cache_key] += 1 # --- if hases[cache_key] % 100 == 0: - printe.output( - f"### <> LoginWrap has bot for ({sitecode}.{family}.org|{username}) count: {hases[cache_key]}", - p=True, + logger.info( + f"### <> LoginWrap has bot for ({sitecode}.{family}.org|{username}) count: {hases[cache_key]}" ) else: login_bot = Login(sitecode, family=family) # --- - printe.output(f"### <> LoginWrap make new bot for ({sitecode}.{family}.org|{username})", p=True) + logger.info(f"### <> LoginWrap make new bot for ({sitecode}.{family}.org|{username})") # --- login_bot.add_users({family: User_tables}, lang=sitecode) # --- diff --git a/newapi/super/login_wrap_new.py b/newapi/super/login_wrap_new.py index 33fea41..ae8746e 100644 --- a/newapi/super/login_wrap_new.py +++ b/newapi/super/login_wrap_new.py @@ -8,11 +8,13 @@ """ +import logging from functools import lru_cache -from ..api_utils import printe from .super_login import Login +logger = logging.getLogger(__name__) + @lru_cache(maxsize=128) def _create_login_bot(sitecode, family, username, password): @@ -23,7 +25,7 @@ def _create_login_bot(sitecode, family, username, password): (i.e., when a new bot is created). This is intentional - on cache hits, the same bot instance is returned without logging. """ - printe.output(f"### <> LoginWrap make new bot for ({sitecode}.{family}.org|{username})", p=True) + logger.info(f"### <> LoginWrap make new bot for ({sitecode}.{family}.org|{username})") # --- login_bot = Login(sitecode, family=family) # --- @@ -47,8 +49,8 @@ def LoginWrap(sitecode, family, bots_login_cache, User_tables): # --- cache_info = _create_login_bot.cache_info() if cache_info.hits > 0 and cache_info.hits % 100 == 0: - printe.output( - f"### <> LoginWrap has bot for ({sitecode}.{family}.org|{username}) count: {cache_info.hits}", p=True + logger.info( + f"### <> LoginWrap has bot for ({sitecode}.{family}.org|{username}) count: {cache_info.hits}" ) # --- # Return bots_login_cache for backward compatibility diff --git a/newapi/super/mwclient/client.py b/newapi/super/mwclient/client.py index 9d0db6e..48e41de 100644 --- a/newapi/super/mwclient/client.py +++ b/newapi/super/mwclient/client.py @@ -6,9 +6,8 @@ import requests from requests.auth import AuthBase, HTTPBasicAuth from requests_oauthlib import OAuth1 -from . import errors, listing -# from .sleep import Sleepers +from . import errors, listing from .util import handle_limit, parse_timestamp, read_in_chunks __version__ = "0.11.0" @@ -124,7 +123,10 @@ def __init__( if reqs and connection_options: print(ValueError("reqs is a deprecated alias of connection_options. Do not specify both.")) if reqs: - warnings.warn("reqs is deprecated in mwclient 1.0.0. Use connection_options instead", DeprecationWarning) + warnings.warn( + "reqs is deprecated in mwclient 1.0.0. Use connection_options instead", + DeprecationWarning, + ) connection_options = reqs self.requests = connection_options or {} self.scheme = scheme @@ -219,7 +221,11 @@ def site_init(self): return meta = self.get( - "query", meta="siteinfo|userinfo", siprop="general|namespaces", uiprop="groups|rights", retry_on_error=False + "query", + meta="siteinfo|userinfo", + siprop="general|namespaces", + uiprop="groups|rights", + retry_on_error=False, ) # Extract site info @@ -389,7 +395,7 @@ def api(self, action, http_method="POST", *args, **kwargs): return info def log_error(self, result, action, params=None) -> None: - logger.error(f"Error occurred: {result}, Action: {action}, Params: {params}") + logger.error(f"API call result: {result}, action: {action}, params: {params}") def handle_api_result(self, info, kwargs=None, sleeper=None): """Checks the given API response, raising an appropriate exception or sleeping if @@ -424,7 +430,10 @@ def handle_api_result(self, info, kwargs=None, sleeper=None): logger.warning(warning["*"]) if "error" in info: - if info["error"].get("code") in {"internal_api_error_DBConnectionError", "internal_api_error_DBQueryError"}: + if info["error"].get("code") in { + "internal_api_error_DBConnectionError", + "internal_api_error_DBQueryError", + }: # sleeper.sleep() return False @@ -432,7 +441,7 @@ def handle_api_result(self, info, kwargs=None, sleeper=None): if info["error"].get("code") == "mwoauth-invalid-authorization" and "Nonce already used" in info[ "error" ].get("info"): - logger.warning("Retrying due to nonce error, see" "https://phabricator.wikimedia.org/T106066") + logger.warning("Retrying due to nonce error, seehttps://phabricator.wikimedia.org/T106066") # sleeper.sleep() return False @@ -517,7 +526,7 @@ def raw_call(self, script, data, files=None, retry_on_error=True, http_method="P stream = self.connection.request(http_method, url, **args) if stream.headers.get("x-database-lag"): wait_time = int(stream.headers.get("retry-after")) - logger.warning("Database lag exceeds max lag. " f"Waiting for {wait_time} seconds, maxlag:{maxlag}") + logger.warning(f"Database lag exceeds max lag. Waiting for {wait_time} seconds, maxlag:{maxlag}") # fall through to the sleep elif stream.status_code == 200: return stream.text @@ -527,14 +536,18 @@ def raw_call(self, script, data, files=None, retry_on_error=True, http_method="P if not retry_on_error: stream.raise_for_status() logger.warning( - "Received {status} response: {text}. " - "Retrying in a moment.".format(status=stream.status_code, text=stream.text) + "Received {status} response: {text}. Retrying in a moment.".format( + status=stream.status_code, text=stream.text + ) ) toraise = "stream" # fall through to the sleep return stream.text - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as err: + except ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + ) as err: # In the event of a network problem # (e.g. DNS failure, refused connection, etc), # Requests will raise a ConnectionError exception. @@ -667,8 +680,7 @@ def require(self, major, minor, revision=None, raise_error=True): elif raise_error: print( errors.MediaWikiVersionError( - "Requires version {required[0]}.{required[1]}, " - "current version is {current[0]}.{current[1]}".format( + "Requires version {required[0]}.{required[1]}, current version is {current[0]}.{current[1]}".format( required=(major, minor), current=(self.version[:2]) ) ) @@ -705,7 +717,14 @@ def email(self, user, text, subject, cc=False): token = self.get_token("email") try: - info = self.post("emailuser", target=user, subject=subject, text=text, ccme=cc, token=token) + info = self.post( + "emailuser", + target=user, + subject=subject, + text=text, + ccme=cc, + token=token, + ) except errors.APIError as e: if e.args[0] == "noemail": print(errors.NoSpecifiedEmail(user, e.args[1])) @@ -887,7 +906,6 @@ def get_token(self, type, force=False, title=None): self.tokens[type] = "0" if self.tokens.get(type, "0") == "0" or force: - if self.version is None or self.version[:2] >= (1, 24): # We use raw_api() rather than api() because api() is adding "userinfo" # to the query and this raises a readapideniederror if the wiki is read @@ -1010,7 +1028,6 @@ def upload( postdata = predata files = None if file is not None: - # Workaround for https://github.com/mwclient/mwclient/issues/65 # ---------------------------------------------------------------- # Since the filename in Content-Disposition is not interpreted, @@ -1109,7 +1126,15 @@ def chunk_upload(self, file, filename, ignorewarnings, comment, text): params["text"] = text return self.post("upload", **params) - def parse(self, text=None, title=None, page=None, prop=None, redirects=False, mobileformat=False): + def parse( + self, + text=None, + title=None, + page=None, + prop=None, + redirects=False, + mobileformat=False, + ): """Parses the given content and returns parser output. Args: @@ -1226,7 +1251,13 @@ def allpages( ) ) return listing.List.get_list(generator)( - self, "allpages", "ap", max_items=max_items, api_chunk_size=api_chunk_size, return_values="title", **kwargs + self, + "allpages", + "ap", + max_items=max_items, + api_chunk_size=api_chunk_size, + return_values="title", + **kwargs, ) def allimages( @@ -1290,13 +1321,24 @@ def alllinks( pfx = listing.List.get_prefix("al", generator) kwargs = dict( listing.List.generate_kwargs( - pfx, ("from", start), ("to", end), prefix=prefix, prop=prop, namespace=namespace + pfx, + ("from", start), + ("to", end), + prefix=prefix, + prop=prop, + namespace=namespace, ) ) if unique: kwargs[pfx + "unique"] = "1" return listing.List.get_list(generator)( - self, "alllinks", "al", max_items=max_items, api_chunk_size=api_chunk_size, return_values="title", **kwargs + self, + "alllinks", + "al", + max_items=max_items, + api_chunk_size=api_chunk_size, + return_values="title", + **kwargs, ) def allcategories( @@ -1316,7 +1358,12 @@ def allcategories( pfx = listing.List.get_prefix("ac", generator) kwargs = dict(listing.List.generate_kwargs(pfx, ("from", start), ("to", end), prefix=prefix, dir=dir)) return listing.List.get_list(generator)( - self, "allcategories", "ac", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs + self, + "allcategories", + "ac", + max_items=max_items, + api_chunk_size=api_chunk_size, + **kwargs, ) def allusers( @@ -1349,7 +1396,14 @@ def allusers( activeusers=activeusers, ) ) - return listing.List(self, "allusers", "au", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + return listing.List( + self, + "allusers", + "au", + max_items=max_items, + api_chunk_size=api_chunk_size, + **kwargs, + ) def blocks( self, @@ -1392,7 +1446,14 @@ def blocks( kwargs = dict( listing.List.generate_kwargs("bk", start=start, end=end, dir=dir, ids=ids, users=users, prop=prop) ) - return listing.List(self, "blocks", "bk", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + return listing.List( + self, + "blocks", + "bk", + max_items=max_items, + api_chunk_size=api_chunk_size, + **kwargs, + ) def deletedrevisions( self, @@ -1408,10 +1469,24 @@ def deletedrevisions( # TODO: Fix (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) kwargs = dict(listing.List.generate_kwargs("dr", start=start, end=end, dir=dir, namespace=namespace, prop=prop)) - return listing.List(self, "deletedrevs", "dr", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + return listing.List( + self, + "deletedrevs", + "dr", + max_items=max_items, + api_chunk_size=api_chunk_size, + **kwargs, + ) def exturlusage( - self, query, prop=None, protocol="http", namespace=None, limit=None, max_items=None, api_chunk_size=None + self, + query, + prop=None, + protocol="http", + namespace=None, + limit=None, + max_items=None, + api_chunk_size=None, ): r"""Retrieve the list of pages that link to a particular domain or URL, as a generator. @@ -1438,7 +1513,14 @@ def exturlusage( kwargs = dict( listing.List.generate_kwargs("eu", query=query, prop=prop, protocol=protocol, namespace=namespace) ) - return listing.List(self, "exturlusage", "eu", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + return listing.List( + self, + "exturlusage", + "eu", + max_items=max_items, + api_chunk_size=api_chunk_size, + **kwargs, + ) def logevents( self, @@ -1458,20 +1540,49 @@ def logevents( (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) kwargs = dict( listing.List.generate_kwargs( - "le", prop=prop, type=type, start=start, end=end, dir=dir, user=user, title=title, action=action + "le", + prop=prop, + type=type, + start=start, + end=end, + dir=dir, + user=user, + title=title, + action=action, ) ) - return listing.List(self, "logevents", "le", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + return listing.List( + self, + "logevents", + "le", + max_items=max_items, + api_chunk_size=api_chunk_size, + **kwargs, + ) def checkuserlog( - self, user=None, target=None, limit=None, dir="older", start=None, end=None, max_items=None, api_chunk_size=10 + self, + user=None, + target=None, + limit=None, + dir="older", + start=None, + end=None, + max_items=None, + api_chunk_size=10, ): """Retrieve checkuserlog items as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) kwargs = dict(listing.List.generate_kwargs("cul", target=target, start=start, end=end, dir=dir, user=user)) return listing.NestedList( - "entries", self, "checkuserlog", "cul", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs + "entries", + self, + "checkuserlog", + "cul", + max_items=max_items, + api_chunk_size=api_chunk_size, + **kwargs, ) # def protectedtitles requires 1.15 @@ -1488,7 +1599,14 @@ def random(self, namespace, limit=None, max_items=None, api_chunk_size=20): (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) kwargs = dict(listing.List.generate_kwargs("rn", namespace=namespace)) - return listing.List(self, "random", "rn", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + return listing.List( + self, + "random", + "rn", + max_items=max_items, + api_chunk_size=api_chunk_size, + **kwargs, + ) def recentchanges( self, @@ -1519,7 +1637,14 @@ def recentchanges( toponly="1" if toponly else None, ) ) - return listing.List(self, "recentchanges", "rc", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + return listing.List( + self, + "recentchanges", + "rc", + max_items=max_items, + api_chunk_size=api_chunk_size, + **kwargs, + ) def revisions(self, revids, prop="ids|timestamp|flags|comment|user"): """Get data about a list of revisions. @@ -1540,7 +1665,11 @@ def revisions(self, revids, prop="ids|timestamp|flags|comment|user"): Returns: A list of revisions """ - kwargs = {"prop": "revisions", "rvprop": prop, "revids": "|".join(map(str, revids))} + kwargs = { + "prop": "revisions", + "rvprop": prop, + "revids": "|".join(map(str, revids)), + } revisions = [] pages = self.get("query", **kwargs).get("query", {}).get("pages", {}).values() @@ -1553,7 +1682,14 @@ def revisions(self, revids, prop="ids|timestamp|flags|comment|user"): return revisions def search( - self, search, namespace="0", what=None, redirects=False, limit=None, max_items=None, api_chunk_size=None + self, + search, + namespace="0", + what=None, + redirects=False, + limit=None, + max_items=None, + api_chunk_size=None, ): """Perform a full text search. @@ -1583,7 +1719,14 @@ def search( kwargs = dict(listing.List.generate_kwargs("sr", search=search, namespace=namespace, what=what)) if redirects: kwargs["srredirects"] = "1" - return listing.List(self, "search", "sr", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + return listing.List( + self, + "search", + "sr", + max_items=max_items, + api_chunk_size=api_chunk_size, + **kwargs, + ) def usercontributions( self, @@ -1607,11 +1750,24 @@ def usercontributions( (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) kwargs = dict( listing.List.generate_kwargs( - "uc", user=user, start=start, end=end, dir=dir, namespace=namespace, prop=prop, show=show + "uc", + user=user, + start=start, + end=end, + dir=dir, + namespace=namespace, + prop=prop, + show=show, ) ) return listing.List( - self, "usercontribs", "uc", max_items=max_items, api_chunk_size=api_chunk_size, uselang=uselang, **kwargs + self, + "usercontribs", + "uc", + max_items=max_items, + api_chunk_size=api_chunk_size, + uselang=uselang, + **kwargs, ) def users(self, users, prop="blockinfo|groups|editcount"): @@ -1644,11 +1800,26 @@ def watchlist( (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) kwargs = dict( - listing.List.generate_kwargs("wl", start=start, end=end, namespace=namespace, dir=dir, prop=prop, show=show) + listing.List.generate_kwargs( + "wl", + start=start, + end=end, + namespace=namespace, + dir=dir, + prop=prop, + show=show, + ) ) if allrev: kwargs["wlallrev"] = "1" - return listing.List(self, "watchlist", "wl", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + return listing.List( + self, + "watchlist", + "wl", + max_items=max_items, + api_chunk_size=api_chunk_size, + **kwargs, + ) def expandtemplates(self, text, title=None, generatexml=False): """ @@ -1705,7 +1876,10 @@ def ask(self, query, title=None): offset = 0 while offset is not None: results = self.raw_api( - "ask", query="{query}|offset={offset}".format(query=query, offset=offset), http_method="GET", **kwargs + "ask", + query="{query}|offset={offset}".format(query=query, offset=offset), + http_method="GET", + **kwargs, ) self.handle_api_result(results) # raises APIError on error offset = results.get("query-continue-offset") diff --git a/newapi/super/mwclient/errors.py b/newapi/super/mwclient/errors.py index 963f015..69ff41e 100644 --- a/newapi/super/mwclient/errors.py +++ b/newapi/super/mwclient/errors.py @@ -15,7 +15,6 @@ class MaximumRetriesExceeded(MwClientError): class APIError(MwClientError): - def __init__(self, code, info, kwargs): self.code = code self.info = info @@ -35,7 +34,6 @@ class EditError(MwClientError): class ProtectedPageError(EditError, InsufficientPermission): - def __init__(self, page, code=None, info=None): self.page = page self.code = code @@ -62,9 +60,11 @@ def __str__(self): class LoginError(MwClientError): - def __init__(self, site, code, info): - super(LoginError, self).__init__(site, {"result": code, "reason": info}) # For backwards-compability + super(LoginError, self).__init__( + site, + {"result": code, "reason": info}, # For backwards-compability + ) self.site = site self.code = code self.info = info @@ -78,13 +78,10 @@ class OAuthAuthorizationError(LoginError): class AssertUserFailedError(MwClientError): - def __init__(self): super(AssertUserFailedError, self).__init__( ( - "By default, mwclient protects you from accidentally editing " - "without being logged in. If you actually want to edit without " - "logging in, you can set force_login on the Site object to False." + "By default, mwclient protects you from accidentally editing without being logged in. If you actually want to edit without logging in, you can set force_login on the Site object to False." ) ) @@ -105,13 +102,10 @@ class NoWriteApi(MwClientError): class InvalidResponse(MwClientError): - def __init__(self, response_text=None): super(InvalidResponse, self).__init__( ( - "Did not get a valid JSON response from the server. Check that " - "you used the correct hostname. If you did, the server might " - "be wrongly configured or experiencing temporary problems." + "Did not get a valid JSON response from the server. Check that you used the correct hostname. If you did, the server might be wrongly configured or experiencing temporary problems." ), response_text, ) diff --git a/newapi/super/mwclient/image.py b/newapi/super/mwclient/image.py index 6de17a8..7d38a8f 100644 --- a/newapi/super/mwclient/image.py +++ b/newapi/super/mwclient/image.py @@ -3,14 +3,18 @@ class Image(page.Page): - def __init__(self, site, name, info=None): super(Image, self).__init__( site, name, info, extra_properties={ - "imageinfo": (("iiprop", "timestamp|user|comment|url|size|sha1|metadata|mime|archivename"),) + "imageinfo": ( + ( + "iiprop", + "timestamp|user|comment|url|size|sha1|metadata|mime|archivename", + ), + ) }, ) self.imagerepository = self._info.get("imagerepository", "") @@ -23,7 +27,10 @@ def imagehistory(self): API doc: https://www.mediawiki.org/wiki/API:Imageinfo """ return listing.PageProperty( - self, "imageinfo", "ii", iiprop="timestamp|user|comment|url|size|sha1|metadata|mime|archivename" + self, + "imageinfo", + "ii", + iiprop="timestamp|user|comment|url|size|sha1|metadata|mime|archivename", ) def imageusage( @@ -68,7 +75,13 @@ def duplicatefiles(self, limit=None, max_items=None, api_chunk_size=None): not only specify the API chunk size. """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - return listing.PageProperty(self, "duplicatefiles", "df", max_items=max_items, api_chunk_size=api_chunk_size) + return listing.PageProperty( + self, + "duplicatefiles", + "df", + max_items=max_items, + api_chunk_size=api_chunk_size, + ) def download(self, destination=None): """ @@ -93,4 +106,8 @@ def download(self, destination=None): return self.site.connection.get(url).content def __repr__(self): - return "<%s object '%s' for %s>" % (self.__class__.__name__, self.name, self.site) + return "<%s object '%s' for %s>" % ( + self.__class__.__name__, + self.name, + self.site, + ) diff --git a/newapi/super/mwclient/listing.py b/newapi/super/mwclient/listing.py index d88a606..2a966a8 100644 --- a/newapi/super/mwclient/listing.py +++ b/newapi/super/mwclient/listing.py @@ -135,7 +135,11 @@ def set_iter(self, data): self._iter = iter(data["query"][self.result_member].values()) def __repr__(self): - return "<%s object '%s' for %s>" % (self.__class__.__name__, self.list_name, self.site) + return "<%s object '%s' for %s>" % ( + self.__class__.__name__, + self.list_name, + self.site, + ) @staticmethod def generate_kwargs(_prefix, *args, **kwargs): @@ -200,7 +204,6 @@ def load_chunk(self): class Category(page.Page, GeneratorList): - def __init__(self, site, name, info=None, namespace=None): page.Page.__init__(self, site, name, info) kwargs = {} @@ -210,22 +213,39 @@ def __init__(self, site, name, info=None, namespace=None): GeneratorList.__init__(self, site, "categorymembers", "cm", **kwargs) def __repr__(self): - return "<%s object '%s' for %s>" % (self.__class__.__name__, self.name, self.site) + return "<%s object '%s' for %s>" % ( + self.__class__.__name__, + self.name, + self.site, + ) def members( - self, prop="ids|title", namespace=None, sort="sortkey", dir="asc", start=None, end=None, generator=True + self, + prop="ids|title", + namespace=None, + sort="sortkey", + dir="asc", + start=None, + end=None, + generator=True, ): prefix = self.get_prefix("cm", generator) kwargs = dict( self.generate_kwargs( - prefix, prop=prop, namespace=namespace, sort=sort, dir=dir, start=start, end=end, title=self.name + prefix, + prop=prop, + namespace=namespace, + sort=sort, + dir=dir, + start=start, + end=end, + title=self.name, ) ) return self.get_list(generator)(self.site, "categorymembers", "cm", **kwargs) class PageList(GeneratorList): - def __init__(self, site, prefix=None, start=None, namespace=0, redirects="all", end=None): self.namespace = namespace @@ -301,7 +321,6 @@ def guess_namespace(self, name): class PageProperty(List): - def __init__(self, page, prop, prefix, *args, **kwargs): super(PageProperty, self).__init__(page.site, prop, prefix, titles=page.name, *args, **kwargs) self.page = page @@ -316,14 +335,12 @@ def set_iter(self, data): class PagePropertyGenerator(GeneratorList): - def __init__(self, page, prop, prefix, *args, **kwargs): super(PagePropertyGenerator, self).__init__(page.site, prop, prefix, titles=page.name, *args, **kwargs) self.page = page class RevisionsIterator(PageProperty): - def load_chunk(self): if "rvstartid" in self.args and "rvstart" in self.args: del self.args["rvstart"] diff --git a/newapi/super/mwclient/page.py b/newapi/super/mwclient/page.py index fa66b0d..d1b02bb 100644 --- a/newapi/super/mwclient/page.py +++ b/newapi/super/mwclient/page.py @@ -5,7 +5,6 @@ class Page: - def __init__(self, site, name, info=None, extra_properties=None): if type(name) is type(self): self.__dict__.update(name.__dict__) @@ -78,7 +77,11 @@ def resolve_redirect(self): return target_page def __repr__(self): - return "<%s object '%s' for %s>" % (self.__class__.__name__, self.name, self.site) + return "<%s object '%s' for %s>" % ( + self.__class__.__name__, + self.name, + self.site, + ) @staticmethod def strip_namespace(title): @@ -273,7 +276,15 @@ def touch(self): return self.append("") - def move(self, new_title, reason="", move_talk=True, no_redirect=False, move_subpages=False, ignore_warnings=False): + def move( + self, + new_title, + reason="", + move_talk=True, + no_redirect=False, + move_subpages=False, + ignore_warnings=False, + ): """Move (rename) page to new_title. If user account is an administrator, specify no_redirect as True to not @@ -391,7 +402,13 @@ def categories(self, generator=True, show=None): return listing.PageProperty(self, "categories", "cl", return_values="title", **kwargs) def embeddedin( - self, namespace=None, filterredir="all", limit=None, generator=True, max_items=None, api_chunk_size=None + self, + namespace=None, + filterredir="all", + limit=None, + generator=True, + max_items=None, + api_chunk_size=None, ): """List pages that transclude the current page. diff --git a/newapi/super/mwclient/util.py b/newapi/super/mwclient/util.py index fe6d0ec..ec623e4 100644 --- a/newapi/super/mwclient/util.py +++ b/newapi/super/mwclient/util.py @@ -36,16 +36,12 @@ def handle_limit(limit, max_items, api_chunk_size): if limit: if api_chunk_size: warnings.warn( - "limit and api_chunk_size both specified, this is not supported! limit " - "is deprecated, will use value of api_chunk_size", + "limit and api_chunk_size both specified, this is not supported! limit is deprecated, will use value of api_chunk_size", DeprecationWarning, ) else: warnings.warn( - "limit is deprecated as its name and purpose are confusing. use " - "api_chunk_size to set the number of items retrieved from the API at " - "once, and/or max_items to limit the total number of items that will be " - "yielded", + "limit is deprecated as its name and purpose are confusing. use api_chunk_size to set the number of items retrieved from the API at once, and/or max_items to limit the total number of items that will be yielded", DeprecationWarning, ) api_chunk_size = limit diff --git a/newapi/super/params_help.py b/newapi/super/params_help.py index d666b7a..1de88c9 100644 --- a/newapi/super/params_help.py +++ b/newapi/super/params_help.py @@ -5,9 +5,10 @@ """ import json +import logging import sys -from ..api_utils.except_err import exception_err +logger = logging.getLogger(__name__) class PARAMS_HELPS: @@ -63,7 +64,7 @@ def parse_data(self, req0) -> dict: return data except Exception as e: - exception_err(e) + logger.exception(e) text = str(req0.text).strip() valid_text = text.startswith("{") and text.endswith("}") @@ -75,6 +76,7 @@ def parse_data(self, req0) -> dict: data = json.loads(text) return data except Exception as e: - exception_err(e, self.url_o_print) + logger.exception(e) + logger.exception(self.url_o_print) return {} diff --git a/newapi/super/super_login.py b/newapi/super/super_login.py index 893dce6..73d2c81 100644 --- a/newapi/super/super_login.py +++ b/newapi/super/super_login.py @@ -13,16 +13,18 @@ # ---- """ + import copy +import logging import sys import time import urllib.parse -from ..api_utils import printe -from ..api_utils.except_err import warn_err from ..api_utils.user_agent import default_user_agent from .handel_errors import HANDEL_ERRORS +logger = logging.getLogger(__name__) + # if "nomwclient" in sys.argv: # from .bot import LOGIN_HELPS # else: @@ -81,7 +83,7 @@ def p_url(self, params): no_remove = ["titles", "title"] # --- pams2 = { - k: v[:100] if isinstance(v, str) and len(v) > 100 and k not in no_remove else v + k: (v[:100] if isinstance(v, str) and len(v) > 100 and k not in no_remove else v) for k, v in params.items() if k not in no_url } @@ -94,7 +96,7 @@ def p_url(self, params): urls_prints[self.url_o_print] += 1 urls_prints["all"] += 1 # --- - printe.output(f"c: {urls_prints[self.url_o_print]}/{urls_prints['all']}\t {self.url_o_print}") + logger.info(f"c: {urls_prints[self.url_o_print]}/{urls_prints['all']}\t {self.url_o_print}") def make_response(self, params, files=None, timeout=30, do_error=True): """ @@ -149,7 +151,16 @@ def filter_params(self, params): def post(self, params, Type="get", addtoken=False, CSRF=True, files=None): return self.post_params(params, Type=Type, addtoken=addtoken, GET_CSRF=CSRF, files=files) - def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=None, do_error=False, max_retry=0): + def post_params( + self, + params, + Type="get", + addtoken=False, + GET_CSRF=True, + files=None, + do_error=False, + max_retry=0, + ): """ Make a POST request to the API endpoint with authentication token. """ @@ -181,7 +192,7 @@ def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=N self.r3_token = self.make_new_r3_token() if not self.r3_token: - printe.output(warn_err('<> self.r3_token == "" ')) + logger.error('<> self.r3_token == "" ') params["token"] = self.r3_token @@ -192,7 +203,7 @@ def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=N data = self.make_response(params, files=files, do_error=do_error) if not data: - printe.output("<> super_login(post): not data. return {}.") + logger.info("<> super_login(post): not data. return {}.") return {} # --- error = data.get("error", {}) @@ -203,10 +214,10 @@ def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=N # code = error.get("code", "") # --- if do_error: - printe.output(f"<> super_login(post): error: {error}") + logger.error(f"<> super_login(post): error: {error}") # --- if Invalid == "Invalid CSRF token.": - printe.output(f'<> ** error "Invalid CSRF token.".\n{self.r3_token} ') + logger.info(f'<> ** error "Invalid CSRF token.".\n{self.r3_token} ') if GET_CSRF: # --- self.r3_token = self.make_new_r3_token() @@ -218,9 +229,9 @@ def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=N if error_code == "maxlag" and max_retry < 4: lage = int(error.get("lag", "0")) # --- - printe.test_print(params) + logger.debug(params) # --- - printe.output(f"<>post_params: <> {lage=} {max_retry=}, sleep: {lage + 1}") + logger.info(f"<>post_params: <> {lage=} {max_retry=}, sleep: {lage + 1}") # --- time.sleep(lage + 1) # --- @@ -231,16 +242,24 @@ def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=N return self.post_params(params, Type=Type, addtoken=addtoken, max_retry=max_retry + 1) # --- if "printdata" in sys.argv: - printe.output(data) + logger.info(data) return data def post_continue( - self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None + self, + params, + action, + _p_="pages", + p_empty=None, + Max=500000, + first=False, + _p_2="", + _p_2_empty=None, ): # --- - printe.test_print("_______________________") - printe.test_print(f"post_continue, start. {action=}, {_p_=}") + logger.debug("_______________________") + logger.debug(f", start. {action=}, {_p_=}") # --- if not isinstance(Max, int) and Max.isdigit(): Max = int(Max) @@ -265,16 +284,16 @@ def post_continue( # --- if continue_params: # params = {**params, **continue_params} - printe.test_print("continue_params:") + logger.debug("continue_params:") for k, v in continue_params.items(): params2[k] = v # params2.update(continue_params) - printe.test_print(params2) + logger.debug(params2) # --- json1 = self.post_params(params2) # --- if not json1: - printe.test_print("post_continue, json1 is empty. break") + logger.debug(", json1 is empty. break") break # --- continue_params = {} @@ -282,8 +301,8 @@ def post_continue( if action == "wbsearchentities": data = json1.get("search", []) # --- - # printe.test_print("wbsearchentities json1: ") - # printe.test_print(str(json1)) + # logger.debug("wbsearchentities json1: ") + # logger.debug(str(json1)) # --- # search_continue = json1.get("search-continue") # --- @@ -303,13 +322,13 @@ def post_continue( data = data.get(_p_2, _p_2_empty) # --- if not data: - printe.test_print("post continue, data is empty. break") + logger.debug("post continue, data is empty. break") break # --- - printe.test_print(f"post continue, len:{len(data)}, all: {len(results)}") + logger.debug(f"post continue, len:{len(data)}, all: {len(results)}") # --- if Max <= len(results) and len(results) > 1: - printe.test_print(f"post continue, {Max=} <= {len(results)=}. break") + logger.debug(f"post continue, {Max=} <= {len(results)=}. break") break # --- if isinstance(results, list): @@ -320,6 +339,6 @@ def post_continue( print(f"{type(data)=}") results = {**results, **data} # --- - printe.test_print(f"post continue, {len(results)=}") + logger.debug(f"post continue, {len(results)=}") # --- return results diff --git a/requirements.in b/requirements.in index 44e8608..48da4e3 100644 --- a/requirements.in +++ b/requirements.in @@ -6,3 +6,4 @@ wikitextparser==0.55.7 mwclient ratelimiter SPARQLWrapper +colorlog diff --git a/tests/TestALL_APIS.py b/tests/TestALL_APIS.py index ac6ecd6..116285d 100644 --- a/tests/TestALL_APIS.py +++ b/tests/TestALL_APIS.py @@ -41,7 +41,7 @@ def test_all_apis_init(mock_dependencies): mock_dependencies["Login"].assert_called_once_with(lang, family=family) mock_dependencies["LoginInstance"].add_users.assert_called_once() - # Verify printe.output was called + # Verify logger.info was called mock_dependencies["printe"].output.assert_called() diff --git a/z_te_sts/test_bot_api.py b/z_te_sts/test_bot_api.py index 124963c..328288c 100644 --- a/z_te_sts/test_bot_api.py +++ b/z_te_sts/test_bot_api.py @@ -2,15 +2,16 @@ Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner """ +import logging import sys import time sys.argv.append("printurl") sys.argv.append("ask") - -from newapi import printe from newapi.page import NEW_API +logger = logging.getLogger(__name__) + class testmybot: def __init__(self): @@ -207,17 +208,17 @@ def start(self): # --- for n, func in defs.items(): name = func.__name__ - printe.output(f"<> start def number {n}, name:{name}:", p=True) + logger.info(f"<> start def number {n}, name:{name}:", p=True) # --- def_name = func.__doc__ - printe.output(f"<> test: {def_name}:", p=True) + logger.info(f"<> test: {def_name}:", p=True) # --- if "tat" in sys.argv: continue # --- result = func() # --- - # printe.output( result ) + # logger.info( result ) # --- if isinstance(result, dict): for n, (na, ta) in enumerate(result.items()): @@ -230,27 +231,27 @@ def start(self): # --- # ta = json.dumps(ta, indent=2, ensure_ascii=False) # --- - printe.output(f"{n}: {na2}: {ta}") + logger.info(f"{n}: {na2}: {ta}") # --- if result == "": raise Exception("result == ''") # --- if "printresult" in sys.argv: if isinstance(result, str): - printe.output(f"result:{result}") + logger.info(f"result:{result}") elif isinstance(result, list): - printe.output(result) + logger.info(result) else: - printe.output(result) + logger.info(result) else: - printe.output("<> add 'printresult' to sys.argv to print result") + logger.info("<> add 'printresult' to sys.argv to print result") # --- if result: - printe.output(f"{len(result)=}", p=True) + logger.info(f"{len(result)=}", p=True) # --- - printe.output("=====================") - printe.output(f"<> test: {def_name} end...") - printe.output("time.sleep(1)") + logger.info("=====================") + logger.info(f"<> test: {def_name} end...") + logger.info("time.sleep(1)") time.sleep(1) diff --git a/z_te_sts/test_ncc_page.py b/z_te_sts/test_ncc_page.py deleted file mode 100644 index f7f089b..0000000 --- a/z_te_sts/test_ncc_page.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" - -import sys - -sys.argv.append("printurl") -sys.argv.append("ask") - -from newapi.ncc_page import CatDepth, MainPage - -title = "Category:Pages_with_script_errors" - -# cat_members = CatDepth(title, depth=0, ns="10", nslist=[], tempyes=[]) - -cat_members = CatDepth(title, sitecode="www", family="nccommons", depth=0, onlyns=10) - -# print(cat_members) -print(f"{len(cat_members)=}") - -page = MainPage( - "Bilateral mesial temporal polymicrogyria (Radiopaedia 76456-88181 Axial SWI)", "www", family="nccommons" -) -# --- -text = page.get_text() -print(f"{len(text)=}")