Merge pull request #7643 from Tribler/feature/AT-scriptcheck

Add Application Tester to `scripttest.yml`

.github/workflows/scripttest.yml

@@ -34,13 +34,17 @@ jobs:
         uses: ./.github/actions/pyenv
         with:
           python-version: ${{inputs.python-version}}
-          requirements: requirements-core.txt
+          requirements: ./scripts/requirements.txt
           custom_cache_key_element: ${{inputs.ref}}
 
-      - name: Set PYTHONPATH
+      - name: Add Tribler to PYTHONPATH
         run: |
           echo "PYTHONPATH=${PYTHONPATH}:./src" >> $GITHUB_ENV
 
+      - name: Add Application Tester to PYTHONPATH
+        run: |
+          echo "PYTHONPATH=${PYTHONPATH}:./scripts/application_tester" >> $GITHUB_ENV
+
       - name: run_bandwidth_crawler.py
         uses: ./.github/actions/timeout
         with:
@@ -56,6 +60,24 @@ jobs:
           command: python ./scripts/exit_node/run_exit_node.py --testnet --fragile
           duration: ${{inputs.duration}}
 
+      # install QT Linux dependencies and Headless Display for the correct work of the Application Tester
+      - name: Install dependencies (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt install pyqt5-dev-tools
+
+      - name: Setup headless display (Linux)
+        if: runner.os == 'Linux'
+        uses: pyvista/setup-headless-display-action@v1
+
+      - name: Application Tester
+        timeout-minutes: 1
+        env:
+          CORE_API_PORT: 20100
+        run: |
+          python ./scripts/application_tester/main.py -p "python src/run_tribler.py" --fragile -d 20
+
       #experiments
 
       - name: hidden_peer_discovery.py

.gitignore

@@ -98,4 +98,4 @@ _build/
 # this folder is dedicated to storing local experimental code that don't want to be committed
 local/
 
-scripts/application-tester/output/
+scripts/application_tester/output/

scripts/application-tester/README.md → scripts/application_tester/README.md

@@ -6,7 +6,7 @@ This repository hosts code for the Tribler application tester, which aims to aut
 To run the Tribler application tester, it requires a path to the Tribler executable, for example:
 
 ```
-python scripts/application-tester/main.py -p "python src/run_tribler.py"
+python scripts/application_tester/main.py -p "python src/run_tribler.py"
 ```
 
 Upon start, the application tester will automatically verify whether Tribler has been started correctly and then open a TCP connection to the code injection port.
@@ -17,17 +17,18 @@ You can specify how long Tribler should be tested by specifying a `-d` (duration
 
 ```
 export CORE_API_PORT=20100
-python scripts/application-tester/main.py -p "python src/run_tribler.py" -d 120 
+python scripts/application_tester/main.py -p "python src/run_tribler.py" -d 120 
 ```
 
 To disable all actions and run Tribler idle, specify the `-s` (silent) flag:
 
 ```
 export CORE_API_PORT=20100
-python scripts/application-tester/main.py -p "python src/run_tribler.py" -s
+python scripts/application_tester/main.py -p "python src/run_tribler.py" -s
 ```
 
-Furthermore, there are several flags to increase resource monitoring. See the output of `python scripts/application-tester/main.py -h` for more details.
+Furthermore, there are several flags to increase resource monitoring.
+See the output of `python scripts/application_tester/main.py -h` for more details.
 
 ## Authors
 

scripts/application-tester/main.py → scripts/application_tester/main.py

@@ -1,6 +1,5 @@
 import argparse
 import asyncio
-import logging
 import os
 from asyncio import ensure_future, get_event_loop
 from pathlib import Path
@@ -12,7 +11,7 @@
 from tribler_apptester.logger.logger import init_logger
 
 sentry_sdk.init(
-    os.environ.get('SENTRY_URL', 'https://e489691c2e214c03961e18069a71d76c@sentry.tribler.org/6'),
+    os.environ.get('APPLICATION_TESTER_SENTRY_DSN'),
     traces_sample_rate=1.0,
     ignore_errors=[KeyboardInterrupt],
 )
@@ -27,18 +26,25 @@
     parser.add_argument('--monitordownloads', default=None, type=int, help='monitor the downloads with a specified interval in seconds')
     parser.add_argument('--monitorresources', default=None, type=int, help='monitor the resources with a specified interval in seconds')
     parser.add_argument('--monitoripv8', default=None, type=int, help='monitor IPv8 overlays with a specified interval in seconds')
-    parser.add_argument('--magnetsfile', default=Path("tribler_apptester") / "data" / "torrent_links.txt", type=str, help='specify the location of the file with magnet links')
     parser.add_argument('--fragile', '-f', help='Fail at the first error', action='store_true')
 
     init_logger()
 
     args = parser.parse_args()
 
     loop = asyncio.get_event_loop()
+    executor_kwargs = {}
     if args.fragile:
         make_async_loop_fragile(loop)
 
-    executor = Executor(args)
+        # Modify the base logic of the Executor to quickly receive feedback from the Application Tester.
+        # This feature is employed in PR tests when we aim to assess potential disruptions in script logic due
+        # to changes within Tribler.
+        executor_kwargs['read_config_delay'] = 0
+        executor_kwargs['read_config_attempts'] = 1
+        executor_kwargs['check_process_started_interval'] = 1
+
+    executor = Executor(args, **executor_kwargs)
 
     loop = get_event_loop()
     coro = executor.start()

scripts/application-tester/tribler_apptester/executor.py → scripts/application_tester/tribler_apptester/executor.py

@@ -8,12 +8,12 @@
 import subprocess
 import sys
 import time
-from asyncio import Task, create_task, get_event_loop, sleep, Future, wait_for
+from asyncio import Future, Task, create_task, get_event_loop, sleep, wait_for
 from base64 import b64encode
 from bisect import bisect
 from distutils.version import LooseVersion
 from pathlib import Path
-from random import random, randint, choice
+from random import choice, randint, random
 from typing import Dict, Optional
 
 from tribler.core.config.tribler_config import TriblerConfig
@@ -42,25 +42,25 @@
 from tribler_apptester.utils.osutils import get_appstate_dir, quote_path_with_spaces
 
 CHECK_PROCESS_STARTED_TIMEOUT = 60
-CHECK_PROCESS_STARTED_INTERVAL = 1
 
 ACTIONS_WARMUP_DELAY = 15
 DELAY_BETWEEN_ACTIONS = 5
 
 SHUTDOWN_TIMEOUT = 30
 
 
-
 class Executor(object):
 
-    def __init__(self, args):
+    def __init__(self, args, read_config_delay=2, read_config_attempts=10, check_process_started_interval=5):
         self.args = args
         self.tribler_path = quote_path_with_spaces(args.tribler_executable)
+        self.read_config_delay = read_config_delay
+        self.check_process_started_interval = check_process_started_interval
+        self.read_config_attempts = read_config_attempts
         self.code_port = args.codeport
         self.api_port = int(os.environ.get('CORE_API_PORT'))
         self._logger = logging.getLogger(self.__class__.__name__)
         self.allow_plain_downloads = args.plain
-        self.magnets_file_path = args.magnetsfile
         self.pending_tasks: Dict[bytes, Future] = {}  # Dictionary of pending tasks
         self.probabilities = []
         self.apptester_start_time = time.time()
@@ -149,7 +149,7 @@ async def wait_for_tribler_process_start(self):
                 self._logger.info("Successfully connected to the code executor port")
                 return True
 
-            await sleep(CHECK_PROCESS_STARTED_INTERVAL)
+            await sleep(self.check_process_started_interval)
 
         self._logger.error("Cannot connect to the code executor port in specified time")
         return False
@@ -188,14 +188,14 @@ async def load_tribler_config(self):
         Attempt to load the Tribler config until we have an API key.
         """
 
-        for attempt in range(1, 10):
-            self._logger.info("Attempting to load Tribler config (%d/10)", attempt)
+        for attempt in range(self.read_config_attempts):
+            self._logger.info(f'Attempting to load Tribler config ({attempt + 1}/{self.read_config_attempts})')
 
             # Read the version_history file and derive the current state dir from that
             versions_file_path = get_appstate_dir() / "version_history.json"
             if not versions_file_path.exists():
                 self._logger.info("Version file at %s does not exist, waiting...", versions_file_path)
-                await sleep(2)
+                await sleep(self.read_config_delay)
             else:
                 with open(versions_file_path, "r") as versions_file:
                     json_content = json.loads(versions_file.read())
@@ -209,7 +209,7 @@ async def load_tribler_config(self):
 
                 config = TriblerConfig.load(state_dir=state_dir, file=config_file_path)
                 if not config.api.key:
-                    await sleep(2)
+                    await sleep(self.read_config_delay)
                 else:
                     self.tribler_config = config
                     self._logger.info(f"Loaded API key: {config.api.key}")
@@ -244,8 +244,8 @@ async def do_testing(self):
 
     def determine_probabilities(self):
         self._logger.info("Determining probabilities of actions")
-        with open(Path("tribler_apptester") / "data" / "action_weights.txt", "r") as action_weights_file:
-            content = action_weights_file.read()
+        with open(Path(__file__).parent / "data/action_weights.txt", mode="r", encoding='utf-8') as f:
+            content = f.read()
             for line in content.split('\n'):
                 if len(line) == 0:
                     continue
@@ -429,7 +429,8 @@ def get_random_action(self):
             elif action_name == 'search':
                 action = RandomSearchAction()
             elif action_name == 'start_download':
-                action = StartRandomDownloadAction(self.magnets_file_path)
+                torrent_links = Path(__file__).parent / "data/torrent_links.txt"
+                action = StartRandomDownloadAction(torrent_links)
             elif action_name == 'remove_download':
                 action = RemoveRandomDownloadAction()
             elif action_name == 'explore_download':

scripts/requirements.txt

@@ -0,0 +1,3 @@
+-r ../requirements.txt
+-r ./application_tester/requirements.txt
+-r ./experiments/requirements.txt