Tools/cicd (#119)

* fix flake8 version

* Create dp-agent.yml

* fix linter warnings

* add "black" code formatter

* formatting source code

* add "mypy" for static type checking

* fix type errors

.flake8

@@ -1,4 +1,4 @@
 [flake8]
 max-line-length=120
-ignore=D100,D101,D102,D103,D107,F403,F405
+ignore=D100,D101,D102,D103,D107,F403,F405,W503
 exclude=.git,__pycache__,build,dist,env

.github/workflows/dp-agent.yml

@@ -0,0 +1,67 @@
+name: Deeppavlov agent
+
+on:
+  push:
+    branches: [ "tools/**" ]
+  pull_request:
+    branches: [ "master", "dev" ]
+
+permissions:
+  contents: read
+
+jobs:
+  format:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.7"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r dev_requirements.txt
+    - name: Check code formatting
+      run: |
+        black deeppavlov_agent/ --diff --check
+
+  lint:
+    needs: format
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.7"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r dev_requirements.txt
+    - name: Lint with flake8
+      run: |
+        flake8 deeppavlov_agent/ --count --show-source --statistics
+
+  type_check:
+    needs: lint
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.7"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r dev_requirements.txt
+    - name: Check types with mypy
+      run: |
+        mypy deeppavlov_agent/

.mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+ignore_missing_imports = True

deeppavlov_agent/channels/telegram/bot.py

@@ -14,14 +14,14 @@
 from .utils import MessageResponder
 from urllib.parse import urlparse
 
-config_dir = Path(__file__).resolve().parent / 'config'
+config_dir = Path(__file__).resolve().parent / "config"
 
 logging.basicConfig(
     format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO
 )
 logger = logging.getLogger(__name__)
 
-FILE_SERVER_URL = getenv('FILE_SERVER_URL')
+FILE_SERVER_URL = getenv("FILE_SERVER_URL")
 server_url = urlparse(FILE_SERVER_URL)
 
 
@@ -156,36 +156,43 @@ async def handle_dialog_rating(
             callback_query.from_user.id, message_text, reply_markup=reply_markup
         )
 
-    @dp.message_handler(state="*", content_types=['text', 'photo'])
+    @dp.message_handler(state="*", content_types=["text", "photo"])
     async def handle_message(message: types.Message, state: FSMContext):
         if await state.get_state() == DialogState.active.state:
             message_attrs = {}
-            if message.photo:
+            if message.photo and FILE_SERVER_URL:
                 try:
                     photo = await message.photo[-1].download(BytesIO())
-                    fname = f'{uuid4().hex}.jpg'
+                    fname = f"{uuid4().hex}.jpg"
                     # TODO: make with aiohttp. Maybe remove BytesIO intermediate step. Maybe mobe image logit to agent.
                     # TODO: move file server url definition to the run.py level
-                    resp = requests.post(FILE_SERVER_URL, files={'file': (fname, photo, 'image/jpg')})
+                    resp = requests.post(
+                        FILE_SERVER_URL, files={"file": (fname, photo, "image/jpg")}
+                    )
                     resp.raise_for_status()
-                    download_link = resp.json()['downloadLink']
-                    download_link = urlparse(download_link)._replace(scheme=server_url.scheme,
-                                                                     netloc=server_url.netloc).geturl()
-                    message_attrs['image'] = download_link
+                    download_link = resp.json()["downloadLink"]
+                    download_link = (
+                        urlparse(download_link)
+                        ._replace(scheme=server_url.scheme, netloc=server_url.netloc)
+                        .geturl()
+                    )
+                    message_attrs["image"] = download_link
                 except Exception as e:
                     logger.error(e)
             response_data = await agent.register_msg(
-                utterance=message.text or '',
+                utterance=message.text or "",
                 user_external_id=str(message.from_user.id),
                 user_device_type="telegram",
                 date_time=message.date,
                 location="",
                 channel_type="telegram",
                 require_response=True,
-                message_attrs=message_attrs
+                message_attrs=message_attrs,
             )
             text = response_data["dialog"].utterances[-1].text
-            response_image = response_data["dialog"].utterances[-1].attributes.get("image")
+            response_image = (
+                response_data["dialog"].utterances[-1].attributes.get("image")
+            )
             utterance_id = response_data["dialog"].utterances[-1].utt_id
             reply_markup = responder.utterance_rating_inline_keyboard(utterance_id)
         else:

deeppavlov_agent/channels/telegram/utils.py

@@ -2,13 +2,18 @@
 from string import Template
 from typing import Union
 
-from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton, ReplyKeyboardMarkup
+from aiogram.types import (
+    InlineKeyboardMarkup,
+    InlineKeyboardButton,
+    ReplyKeyboardMarkup,
+)
 from pydantic import BaseModel
 from yaml import load
+
 try:
-    from yaml import CLoader as Loader, CDumper as Dumper
+    from yaml import CLoader as Loader
 except ImportError:
-    from yaml import Loader, Dumper
+    from yaml import Loader  # type: ignore
 
 
 def _load_yml(path):
@@ -87,7 +92,7 @@ def message(self, key: str, **kwargs) -> str:
         return template.safe_substitute(**kwargs)
 
     def dialog_rating_inline_keyboard(
-        self, dialog_id: str, chosen_rating: Union[str, int] = None
+        self, dialog_id: str, chosen_rating: Union[str, int, None] = None
     ) -> InlineKeyboardMarkup:
         """Create inline keyboard with rating buttons. Min and max score are set via config.
         Provide chosen_rating argument if the keyboard is edited after the conversation was rated
@@ -115,7 +120,9 @@ def dialog_rating_inline_keyboard(
 
         return reply_markup
 
-    def utterance_rating_inline_keyboard(self, utterance_id: str) -> InlineKeyboardMarkup:
+    def utterance_rating_inline_keyboard(
+        self, utterance_id: str
+    ) -> InlineKeyboardMarkup:
         """Create inline keyboard with thumbs up/down buttons
 
         Args: