Merge pull request #52 from life4/add-ids

Add post IDs
life4 · Aug 2, 2023 · 14939c6 · 14939c6
2 parents 0134286 + a91f5c0
commit 14939c6
Show file tree

Hide file tree

Showing 32 changed files with 140 additions and 18 deletions.
diff --git a/assign_ids.py b/assign_ids.py
diff --git a/posts/add-note.md b/posts/add-note.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 704
 published: 2022-11-01
 traces:
   - [type: BaseException, method: add_note]

diff --git a/posts/assert-type.md b/posts/assert-type.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 709
 published: 2022-11-29
 traces:
   - [module: typing, function: assert_type]

diff --git a/posts/asyncio-create-task.md b/posts/asyncio-create-task.md
@@ -1,5 +1,6 @@
 ---
 published: 2023-04-18
+id: 724
 author: orsinium
 traces:
   - [module: asyncio, function: create_task]

diff --git a/posts/asyncio-gather.md b/posts/asyncio-gather.md
@@ -1,5 +1,6 @@
 ---
 published: 2023-04-11
+id: 723
 author: orsinium
 traces:
   - [module: asyncio, function: gather]

diff --git a/posts/asyncio-run.md b/posts/asyncio-run.md
@@ -1,5 +1,6 @@
 ---
 published: 2023-03-28
+id: 721
 author: orsinium
 traces:
   - [module: asyncio, function: run]

diff --git a/posts/asyncio-sleep.md b/posts/asyncio-sleep.md
@@ -1,5 +1,6 @@
 ---
 published: 2023-04-04
+id: 722
 author: orsinium
 traces:
   - [module: asyncio, function: sleep]

diff --git a/posts/build-system.md b/posts/build-system.md
@@ -1,5 +1,6 @@
 ---
 published: 2023-02-28
+id: 717
 author: orsinium
 topics:
   - packaging

diff --git a/posts/chain-is.md b/posts/chain-is.md
@@ -1,5 +1,6 @@
 ---
 published: 2022-09-27
+id: 699
 author: orsinium
 traces:
   - [keyword: is]

diff --git a/posts/contextlib-chdir.md b/posts/contextlib-chdir.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 708
 published: 2022-11-22
 traces:
   - [module: contextlib, function: chdir]

diff --git a/posts/dataclass-transform.md b/posts/dataclass-transform.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 710
 published: 2022-12-06
 pep: 681
 python: "3.11"

diff --git a/posts/datetime-fold.md b/posts/datetime-fold.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 727
 published: 2023-07-25
 pep: 495
 python: "3.6"

diff --git a/posts/except-star.md b/posts/except-star.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 706
 published: 2022-11-15
 traces:
   - [{exception: ExceptionGroup}]

diff --git a/posts/exception-group.md b/posts/exception-group.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 705
 published: 2022-11-08
 traces:
   - [{exception: ExceptionGroup}]

diff --git a/posts/generator-scope.md b/posts/generator-scope.md
@@ -1,5 +1,6 @@
 ---
 author: ypankovych
+id: 700
 published: 2022-10-11
 ---
 

diff --git a/posts/isinstance.md b/posts/isinstance.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 714
 published: 2023-01-03
 traces:
   - [{function: isinstance}]

diff --git a/posts/literal-string.md b/posts/literal-string.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 713
 published: 2022-12-27
 pep: 675
 python: "3.11"

diff --git a/posts/math-fsum.md b/posts/math-fsum.md
@@ -1,5 +1,6 @@
 ---
 published: 2023-03-14
+id: 719
 author: orsinium
 traces:
   - [module: math, function: fsum]

diff --git a/posts/os-curdir.md b/posts/os-curdir.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 701
 published: 2022-10-18
 traces:
   - [module: os, function: curdir]

diff --git a/posts/pyproject-toml.md b/posts/pyproject-toml.md
@@ -1,5 +1,6 @@
 ---
 published: 2023-02-21
+id: 716
 author: orsinium
 topics:
   - packaging

diff --git a/posts/python-3-11.md b/posts/python-3-11.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 702
 published: 2022-10-24
 topics:
   - news

diff --git a/posts/reveal-type.md b/posts/reveal-type.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 712
 published: 2022-12-20
 topics:
   - typing

diff --git a/posts/string-template.md b/posts/string-template.md
@@ -1,5 +1,6 @@
 ---
 published: 2022-09-20
+id: 698
 author: orsinium
 traces:
   - [module: string, type: Template]

diff --git a/posts/tomllib.md b/posts/tomllib.md
@@ -1,5 +1,6 @@
 ---
 published: 2023-03-07
+id: 718
 author: orsinium
 traces:
   - [module: tomllib]

diff --git a/posts/traceback-col.md b/posts/traceback-col.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 703
 published: 2022-10-25
 pep: 657
 python: "3.11"

diff --git a/posts/typing-self.md b/posts/typing-self.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 711
 published: 2022-12-03
 traces:
   - [module: typing, type: Self]

diff --git a/posts/unittest-mock-seal.md b/posts/unittest-mock-seal.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 728
 published: 2023-08-01
 traces:
   - [module: unittest.mock, function: seal]

diff --git a/posts/warnings.md b/posts/warnings.md
@@ -1,5 +1,6 @@
 ---
 published: 2022-09-13
+id: 697
 author: orsinium
 traces:
   - [module: warnings]

diff --git a/posts/wheel.md b/posts/wheel.md
@@ -1,5 +1,6 @@
 ---
 published: 2023-02-14
+id: 715
 author: orsinium
 topics:
   - packaging

diff --git a/posts/zoneinfo.md b/posts/zoneinfo.md
@@ -1,5 +1,6 @@
 ---
 author: orsinium
+id: 726
 published: 2023-07-18
 pep: 615
 python: "3.6"

diff --git a/sdk/commands/__init__.py b/sdk/commands/__init__.py
@@ -1,3 +1,4 @@
+from ._add_ids import AddIDsCommand
 from ._check_all import CheckAllCommand
 from ._command import Command
 from ._html import HTMLCommand
@@ -10,6 +11,7 @@
 __all__ = ['Command', 'COMMANDS']
 
 COMMANDS = (
+    AddIDsCommand,
     HTMLCommand,
     RunCodeCommand,
     ScheduleCommand,

diff --git a/sdk/commands/_add_ids.py b/sdk/commands/_add_ids.py
@@ -0,0 +1,109 @@
+
+from __future__ import annotations
+
+import asyncio
+import os
+import re
+from functools import cached_property
+from pathlib import Path
+
+from telethon import TelegramClient
+
+from ..post import Post
+from ._command import Command
+
+
+CHANNEL = 'pythonetc'
+REX_URL = re.compile(r'https?\:[a-zA-Z0-9\-\/\.\(]+')
+REX_PYTHON = re.compile(r'3\.[0-9]{1,2}')
+REX_QNAME = re.compile(r'[a-z]{2,}\.[a-zA-Z0-9]{2,}')
+URLs = tuple[str, ...]
+
+
+class AddIDsCommand(Command):
+    """Add Telegram post IDs to all posts that don't have it.
+    """
+    name = 'add-ids'
+
+    def run(self) -> int:
+        if 'API_ID' not in os.environ:
+            self.warn(
+                'API_ID and API_HASH env vars required, '
+                'you can get them at https://my.telegram.org/apps',
+            )
+            return 1
+        asyncio.run(self._run())
+        return 0
+
+    async def _run(self) -> None:
+        self.print('reading posts...')
+        paths = self._get_paths()
+        self.print('fetching IDs...')
+        async with self._client:
+            ids = await self._get_ids()
+        self.print('setting IDs...')
+        for keyword, id in ids.items():
+            path = paths.get(keyword)
+            if path is None:
+                continue
+            content = path.read_text(encoding='utf-8')
+            lines = content.splitlines()
+            lines.insert(2, f'id: {id}')
+            new_content = '\n'.join(lines)
+            new_content = new_content.rstrip() + '\n'
+            assert new_content != content
+            path.write_text(new_content, encoding='utf-8')
+            self.print(f'added ID for {path.name}')
+
+    def _get_paths(self) -> dict[URLs, Path]:
+        paths: dict[URLs, Path] = {}
+        for path in sorted(Path('posts').iterdir()):
+            if path.suffix != '.md':
+                continue
+            post = Post.from_path(path)
+            if post.id is not None:
+                continue
+            keywords = self._get_keywords(post.md_content)
+            if not keywords:
+                continue
+            if keywords in paths:
+                name1 = path.name
+                name2 = paths[keywords].name
+                msg = f'duplicate set of keywords: {name1} and {name2}'
+                raise RuntimeError(msg)
+            paths[keywords] = path
+        return paths
+
+    async def _get_ids(self) -> dict[URLs, int]:
+        ids = {}
+        async for message in self._client.iter_messages(CHANNEL):
+            if message.text is None:
+                continue
+            keywords = self._get_keywords(message.text)
+            if not keywords:
+                continue
+            if keywords in ids:
+                continue
+            ids[keywords] = message.id
+        return ids
+
+    def _get_keywords(self, text: str) -> URLs:
+        """
+        Get some key components from the text (URLs, qualnames, Python versions)
+        that can be used to uniquely identify a text.
+
+        It allows us to match the same text ignoring changes in formatting
+        or corrected typos.
+        """
+        result = REX_URL.findall(text)
+        result.extend(REX_PYTHON.findall(text))
+        result.extend(REX_QNAME.findall(text))
+        return tuple(result)
+
+    @cached_property
+    def _client(self) -> TelegramClient:
+        return TelegramClient(
+            'bot',
+            api_id=os.environ['API_ID'],
+            api_hash=os.environ['API_HASH'],
+        )