feat: Major update

- cancel, // now remember cancellation permanently
- reactions fix now remembers message content hashes after a restart. This means no more 'Edit message again to evaluate' messages
- implemented simple key-value store for storing various data
  - `tgpy.api.config.get(key: str, default: JSON = None) -> JSON`
  - `tgpy.api.config.set(key: str, value: JSON)`
  - `tgpy.api.config.unset(key: str)`
  - `tgpy.api.config.save()` useful when modifying objects acquired using the .get method in place
- if the `__all__` variable is set in a module, only objects with names in that list are exported (added to variables list)
- `ctx.is_module` is True if the code is executed as a module (on startup)
- `ctx.is_manual_output` can be set to True to prevent the last message edit by TGPy so you can edit it yourself
- cancel, //, restart, ping, update, modules object and .await syntax are now implemented as regular modules in the `std` directory
- restructure the code
- `tgpy.api` module is now used for public API instead of the `tgpy` object
- new public API functions:
  - `async parse_code(text: str) -> ParseResult(is_code: bool, original: str, transformed: str, tree: AST | None)`
  - `parse_tgpy_message(message: Message) -> MessageParseResult(is_tgpy_message: bool, code: str | None, result: str | None)`
  - `async tgpy_eval(code: str, message: Message = None, *, filename: str = None) -> EvalResult(result: Any, output: str)`
  - `apply_code_transformers(code: str) -> str`
- AST transformers. AST transformers are applied after code transformers. API functions:
  - `add_ast_transformer(name: str, transformer)`
  - `async apply_ast_transformers(tree: AST) -> AST`
- exec hooks. Exec hooks are executed before the message is parsed and handled. Exec hooks must have the following signature: `async hook(message: Message, is_edit: bool) -> Message | bool | None`. An exec hook may edit the message using Telegram API methods and/or alter the message in place. If a hook returns Message object or alters it in place, it's used instead of the original Message object during the rest of handling (including calling other hook functions). If a hook returns True or None, execution completes normally. If a hook returns False, the rest of hooks are executed and then the handling stops without further message parsing or evaluating.
Exec hooks API methods:
  - `add_exec_hook(name: str, hook)`
  - `apply_exec_hooks(message: Message, *, is_edit: bool) -> Message | False` returns False if any of the hooks returned False, Message object that should be used instead of the original one otherwise

.gitignore

@@ -9,7 +9,6 @@ dist/
 *.session
 *.session-journal
 data/
-config.py
 config.yaml
 
 guide/site

guide/docs/reference/builtins.md

@@ -39,11 +39,13 @@
 | --- | --- |
 | `#!python ctx.msg` | The message containing the code TGPy is evaluating at the moment |
 
-## TGPy API object
+## TGPy API module
 
-| Object | Description |
-| --- | --- |
-| `#!python tgpy.add_code_transformer(name: str, transformer: Callable[[str], str])` | _To be documented_ |
-| `#!python tgpy.code_transformers` | _To be documented_ |
-| `#!python tgpy.variables` | Dictionary of saved variables |
-| `#!python tgpy.constants` | Dictionary of constants<br>(`tgpy`, `ctx`, `client`) |
+Can be imported with `import tgpy.api`
+
+| Object                                                                                 | Description                                  |
+|----------------------------------------------------------------------------------------|----------------------------------------------|
+| `#!python tgpy.api.add_code_transformer(name: str, transformer: Callable[[str], str])` | _To be documented_                           |
+| `#!python tgpy.api.code_transformers`                                                  | _To be documented_                           |
+| `#!python tgpy.api.variables`                                                          | Dictionary of saved variables                |
+| `#!python tgpy.api.constants`                                                          | Dictionary of constants<br>(`ctx`, `client`) |

tgpy/__init__.py

@@ -1,29 +1,26 @@
 import logging
 
-from rich.logging import RichHandler
 from telethon import TelegramClient
 
-from tgpy.api import API
-from tgpy.app_config import Config
-from tgpy.console import console
 from tgpy.context import Context
 from tgpy.version import __version__
 
 logging.basicConfig(
-    level=logging.INFO, format='%(message)s', datefmt="[%X]", handlers=[RichHandler()]
+    format='[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    level=logging.INFO,
 )
 logging.getLogger('telethon').setLevel(logging.WARNING)
 
 
 class App:
-    config: Config = None
-    client: TelegramClient = None
-    api: API = None
+    client: TelegramClient
     ctx: Context
 
     def __init__(self):
         self.ctx = Context()
-        self.api = API()
 
 
 app = App()
+
+__all__ = ['App', 'app']

tgpy/_core/eval_message.py

@@ -0,0 +1,40 @@
+from telethon.errors import MessageIdInvalidError
+from telethon.tl.custom import Message
+
+from tgpy import app
+from tgpy._core import message_design
+from tgpy._core.utils import convert_result, format_traceback
+from tgpy.api.tgpy_eval import tgpy_eval
+
+
+async def eval_message(code: str, message: Message) -> Message | None:
+    await message_design.edit_message(message, code, 'Running...')
+
+    # noinspection PyBroadException
+    try:
+        eval_result = await tgpy_eval(code, message, filename=None)
+    except Exception:
+        result = 'Error occurred'
+        output = ''
+        exc = ''.join(format_traceback())
+    else:
+        if app.ctx.is_manual_output:
+            return
+        result = convert_result(eval_result.result)
+        output = eval_result.output
+        exc = ''
+
+    try:
+        # noinspection PyProtectedMember
+        return await message_design.edit_message(
+            message,
+            code,
+            result,
+            traceback=exc,
+            output=output,
+        )
+    except MessageIdInvalidError:
+        return None
+
+
+__all__ = ['eval_message']

tgpy/message_design.py → tgpy/_core/message_design.py

@@ -1,7 +1,5 @@
 import sys
 import traceback as tb
-from dataclasses import dataclass
-from typing import Optional
 
 from telethon.tl.custom import Message
 from telethon.tl.types import (
@@ -87,30 +85,13 @@ async def edit_message(
     return await message.edit(text, formatting_entities=entities, link_preview=False)
 
 
-@dataclass
-class MessageParseResult:
-    is_tgpy_message: bool
-    code: Optional[str]
-    result: Optional[str]
-
-
-def get_title_entity(message: Message) -> Optional[MessageEntityTextUrl]:
+def get_title_entity(message: Message) -> MessageEntityTextUrl | None:
     for e in message.entities or []:
         if isinstance(e, MessageEntityTextUrl) and e.url == TITLE_URL:
             return e
     return None
 
 
-def parse_message(message: Message) -> MessageParseResult:
-    e = get_title_entity(message)
-    if not e:
-        return MessageParseResult(False, None, None)
-    msg_text = Utf16CodepointsWrapper(message.raw_text)
-    code = msg_text[: e.offset].strip()
-    result = msg_text[e.offset + e.length :].strip()
-    return MessageParseResult(True, code, result)
-
-
 async def send_error(chat) -> None:
     exc = ''.join(tb.format_exception(*sys.exc_info()))
     if len(exc) > 4000:
@@ -125,7 +106,5 @@ async def send_error(chat) -> None:
 
 __all__ = [
     'edit_message',
-    'MessageParseResult',
-    'parse_message',
     'send_error',
 ]

tgpy/run_code/meval.py → tgpy/_core/meval.py

@@ -3,14 +3,13 @@
 import ast
 import inspect
 from collections import deque
+from copy import deepcopy
 from importlib.abc import SourceLoader
 from importlib.util import module_from_spec, spec_from_loader
 from types import CodeType
 from typing import Any, Iterator
 
-from tgpy import app
-from tgpy.run_code import autoawait
-from tgpy.run_code.utils import apply_code_transformers
+from tgpy.api.parse_code import ParseResult
 
 
 def shallow_walk(node) -> Iterator:
@@ -48,33 +47,16 @@ def get_code(self, _):
         return self.code
 
 
-async def meval(
-    str_code: str, filename: str, globs: dict, saved_variables: dict, **kwargs
+async def _meval(
+    parsed: ParseResult, filename: str, saved_variables: dict, **kwargs
 ) -> (dict, Any):
     kwargs.update(saved_variables)
 
-    # Restore globals later
-    globs = globs.copy()
-
-    # This code saves __name__ and __package into a kwarg passed to the function.
-    # It is set before the users code runs to make sure relative imports work
-    global_args = '_globs'
-    while global_args in globs.keys():
-        # Make sure there's no name collision, just keep prepending _s
-        global_args = '_' + global_args
-    kwargs[global_args] = {}
-    for glob in ['__name__', '__package__']:
-        # Copy data to args we are sending
-        kwargs[global_args][glob] = globs[glob]
-
-    str_code = apply_code_transformers(app, str_code)
-    root = ast.parse(str_code, '', 'exec')
-    autoawait.transformer.visit(root)
-
+    root = deepcopy(parsed.tree)
     ret_name = '_ret'
     ok = False
     while True:
-        if ret_name in globs.keys():
+        if ret_name in kwargs.keys():
             ret_name = '_' + ret_name
             continue
         for node in ast.walk(root):
@@ -89,32 +71,13 @@ async def meval(
     if not code:
         return {}, None
 
-    # globals().update(**<global_args>)
-    glob_copy = ast.Expr(
-        ast.Call(
-            func=ast.Attribute(
-                value=ast.Call(
-                    func=ast.Name(id='globals', ctx=ast.Load()), args=[], keywords=[]
-                ),
-                attr='update',
-                ctx=ast.Load(),
-            ),
-            args=[],
-            keywords=[
-                ast.keyword(arg=None, value=ast.Name(id=global_args, ctx=ast.Load()))
-            ],
-        )
-    )
-    ast.fix_missing_locations(glob_copy)
-    code.insert(0, glob_copy)
-
     # _ret = []
     ret_decl = ast.Assign(
         targets=[ast.Name(id=ret_name, ctx=ast.Store())],
         value=ast.List(elts=[], ctx=ast.Load()),
     )
     ast.fix_missing_locations(ret_decl)
-    code.insert(1, ret_decl)
+    code.insert(0, ret_decl)
 
     # __import__('builtins').locals()
     get_locals = ast.Call(
@@ -205,15 +168,13 @@ async def meval(
 
     # print(ast.unparse(mod))
     comp = compile(mod, filename, 'exec')
-    loader = MevalLoader(str_code, comp, filename)
+    loader = MevalLoader(parsed.original, comp, filename)
     py_module = module_from_spec(spec_from_loader(filename, loader, origin=filename))
     loader.exec_module(py_module)
 
     new_locs, ret = await getattr(py_module, 'tmp')(**kwargs)
     for loc in list(new_locs):
-        if (
-            loc in globs or loc in kwargs or loc == ret_name
-        ) and loc not in saved_variables:
+        if (loc in kwargs or loc == ret_name) and loc not in saved_variables:
             new_locs.pop(loc)
 
     ret = [await el if inspect.isawaitable(el) else el for el in ret]

tgpy/_core/utils.py

@@ -0,0 +1,30 @@
+import sys
+import tokenize
+import traceback
+from io import BytesIO
+
+from telethon.tl import TLObject
+
+
+def convert_result(result):
+    if isinstance(result, TLObject):
+        result = result.stringify()
+
+    return result
+
+
+def format_traceback():
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    exc_traceback = exc_traceback.tb_next.tb_next
+    return traceback.format_exception(exc_type, exc_value, exc_traceback)
+
+
+def tokenize_string(s: str) -> list[tokenize.TokenInfo] | None:
+    try:
+        return list(tokenize.tokenize(BytesIO(s.encode('utf-8')).readline))
+    except (IndentationError, tokenize.TokenError):
+        return None
+
+
+def untokenize_to_string(tokens: list[tokenize.TokenInfo]) -> str:
+    return tokenize.untokenize(tokens).decode('utf-8')