From df8e57b773def3d2f3cd7bc4bc094985c4f248a2 Mon Sep 17 00:00:00 2001 From: Vladyslav Hnatiuk Date: Sat, 6 Sep 2025 13:16:22 +0200 Subject: [PATCH 01/11] Fix name conflict in runner context: introduce action cache and action handler cache inside of it, because handlers from different actions can have the same name --- .../src/finecode_extension_runner/__main__.py | 2 +- .../_services/run_action.py | 65 ++++++----- .../src/finecode_extension_runner/context.py | 14 +-- .../finecode_extension_runner/di/resolver.py | 2 +- .../src/finecode_extension_runner/domain.py | 13 +++ .../src/finecode_extension_runner/services.py | 103 +++++++----------- 6 files changed, 97 insertions(+), 102 deletions(-) diff --git a/finecode_extension_runner/src/finecode_extension_runner/__main__.py b/finecode_extension_runner/src/finecode_extension_runner/__main__.py index c55e863..2149a65 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/__main__.py +++ b/finecode_extension_runner/src/finecode_extension_runner/__main__.py @@ -1,4 +1,4 @@ from finecode_extension_runner import cli if __name__ == "__main__": - cli.cli() + cli.main() diff --git a/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py b/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py index 8d30970..3456378 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py +++ b/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py @@ -79,16 +79,19 @@ async def run_action( # returned. (experimental) # - execution of handlers can be concurrent or sequential. But executions of handler # on iterable payloads(single parts) are always concurrent. + action_name = request.action_name try: - action_exec_info = global_state.runner_context.action_exec_info_by_name[ - request.action_name - ] + action_cache = global_state.runner_context.action_cache_by_name[action_name] except KeyError: + action_cache = domain.ActionCache() + global_state.runner_context.action_cache_by_name[action_name] = action_cache + + if action_cache.exec_info is not None: + action_exec_info = action_cache.exec_info + else: action_exec_info = create_action_exec_info(action) - global_state.runner_context.action_exec_info_by_name[request.action_name] = ( - action_exec_info - ) + action_cache.exec_info = action_exec_info # TODO: catch validation errors payload: code_action.RunActionPayload | None = None @@ -140,6 +143,7 @@ async def run_action( payload=payload, run_context=run_context, run_id=run_id, + action_cache=action_cache, action_context=action_context, action_exec_info=action_exec_info, runner_context=runner_context, @@ -223,6 +227,7 @@ async def run_action( payload=payload, run_context=run_context, run_id=run_id, + action_cache=action_cache, action_context=action_context, action_exec_info=action_exec_info, runner_context=runner_context, @@ -253,6 +258,7 @@ async def run_action( payload=payload, run_context=run_context, run_id=run_id, + action_cache=action_cache, action_context=action_context, action_exec_info=action_exec_info, runner_context=runner_context, @@ -379,19 +385,37 @@ async def execute_action_handler( run_context: code_action.RunActionContext | None, run_id: int, action_exec_info: domain.ActionExecInfo, + action_cache: domain.ActionCache, action_context: code_action.ActionContext, runner_context: context.RunnerContext, ) -> code_action.RunActionResult: logger.trace(f"R{run_id} | Run {handler.name} on {str(payload)[:100]}...") + if handler.name in action_cache.handler_cache_by_name: + handler_cache = action_cache.handler_cache_by_name[handler.name] + else: + handler_cache = domain.ActionHandlerCache() + action_cache.handler_cache_by_name[handler.name] = handler_cache + start_time = time.time_ns() execution_result: code_action.RunActionResult | None = None + + handler_global_config = runner_context.project.action_handler_configs.get( + handler.source, None + ) + handler_raw_config = {} + if handler_global_config is not None: + handler_raw_config = handler_global_config + if handler_raw_config == {}: + # still empty, just assign + handler_raw_config = handler.config + else: + # not empty anymore, deep merge + handler_config_merger.merge(handler_raw_config, handler.config) - if handler.name in runner_context.action_handlers_instances_by_name: - handler_instance = runner_context.action_handlers_instances_by_name[ - handler.name - ] + if handler_cache.instance is not None: + handler_instance = handler_cache.instance handler_run_func = handler_instance.run - exec_info = runner_context.action_handlers_exec_info_by_name[handler.name] + exec_info = handler_cache.exec_info logger.trace( f"R{run_id} | Instance of action handler {handler.name} found in cache" ) @@ -411,19 +435,6 @@ async def execute_action_handler( f"Import of action handler '{handler.name}' failed(Run {run_id}): {handler.source}" ) - handler_global_config = runner_context.project.action_handler_configs.get( - handler.source, None - ) - handler_raw_config = {} - if handler_global_config is not None: - handler_raw_config = handler_global_config - if handler_raw_config == {}: - # still empty, just assign - handler_raw_config = handler.config - else: - # not empty anymore, deep merge - handler_config_merger.merge(handler_raw_config, handler.config) - def get_handler_config(param_type): # TODO: validation errors return param_type(**handler_raw_config) @@ -437,7 +448,7 @@ def get_process_executor(param_type): exec_info = domain.ActionHandlerExecInfo() # save immediately in context to be able to shutdown it if the first execution # is interrupted by stopping ER - runner_context.action_handlers_exec_info_by_name[handler.name] = exec_info + handler_cache.exec_info = exec_info if inspect.isclass(action_handler): args = resolve_func_args_with_di( func=action_handler.__init__, @@ -453,9 +464,7 @@ def get_process_executor(param_type): exec_info.lifecycle = args["lifecycle"] handler_instance = action_handler(**args) - runner_context.action_handlers_instances_by_name[handler.name] = ( - handler_instance - ) + handler_cache.instance = handler_instance handler_run_func = handler_instance.run else: handler_run_func = action_handler diff --git a/finecode_extension_runner/src/finecode_extension_runner/context.py b/finecode_extension_runner/src/finecode_extension_runner/context.py index baf4844..88838a8 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/context.py +++ b/finecode_extension_runner/src/finecode_extension_runner/context.py @@ -1,20 +1,12 @@ -from dataclasses import dataclass, field +from __future__ import annotations -from finecode_extension_api import code_action +from dataclasses import dataclass, field from finecode_extension_runner import domain @dataclass class RunnerContext: project: domain.Project - action_exec_info_by_name: dict[str, domain.ActionExecInfo] = field( - default_factory=dict - ) - action_handlers_instances_by_name: dict[str, code_action.ActionHandler] = field( - default_factory=dict - ) - action_handlers_exec_info_by_name: dict[str, domain.ActionHandlerExecInfo] = field( - default_factory=dict - ) + action_cache_by_name: dict[str, domain.ActionCache] = field(default_factory=dict) # don't overwrite, only append and remove docs_owned_by_client: list[str] = field(default_factory=list) diff --git a/finecode_extension_runner/src/finecode_extension_runner/di/resolver.py b/finecode_extension_runner/src/finecode_extension_runner/di/resolver.py index f9d10e2..a3f11c9 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/di/resolver.py +++ b/finecode_extension_runner/src/finecode_extension_runner/di/resolver.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, Type, TypeVar +from typing import Type, TypeVar from finecode_extension_api import code_action diff --git a/finecode_extension_runner/src/finecode_extension_runner/domain.py b/finecode_extension_runner/src/finecode_extension_runner/domain.py index c422f5a..4f91569 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/domain.py +++ b/finecode_extension_runner/src/finecode_extension_runner/domain.py @@ -1,6 +1,7 @@ from __future__ import annotations import enum +import dataclasses import typing from pathlib import Path @@ -76,6 +77,18 @@ class ActionHandlerExecInfoStatus(enum.Enum): SHUTDOWN = enum.auto() +@dataclasses.dataclass +class ActionCache: + exec_info: ActionExecInfo | None = None + handler_cache_by_name: dict[str, ActionHandlerCache] = dataclasses.field(default_factory=dict) + + +@dataclasses.dataclass +class ActionHandlerCache: + instance: code_action.ActionHandler | None = None + exec_info: ActionHandlerExecInfo | None = None + + class TextDocumentInfo: def __init__(self, uri: str, version: str, text: str) -> None: self.uri = uri diff --git a/finecode_extension_runner/src/finecode_extension_runner/services.py b/finecode_extension_runner/src/finecode_extension_runner/services.py index 5d683a4..5a695b0 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/services.py +++ b/finecode_extension_runner/src/finecode_extension_runner/services.py @@ -1,19 +1,12 @@ -import asyncio -import collections.abc import importlib -import inspect import sys -import time import types import typing from pathlib import Path from loguru import logger -from pydantic.dataclasses import dataclass as pydantic_dataclass -from finecode_extension_api import code_action, textstyler -from finecode_extension_runner import context, domain, global_state, run_utils, schemas -from finecode_extension_runner._services import run_action as run_action_module +from finecode_extension_runner import context, domain, global_state, schemas from finecode_extension_runner._services.run_action import ( ActionFailedException, StopWithResponse, @@ -96,49 +89,37 @@ def reload_action(action_name: str) -> None: # TODO: raise error return - actions_to_remove: list[str] = [ - action_name, - *[handler.name for handler in action_obj.handlers], - ] + if action_name in global_state.runner_context.action_cache_by_name: + action_cache = global_state.runner_context.action_cache_by_name[action_name] - for _action_name in actions_to_remove: - try: - del global_state.runner_context.action_handlers_instances_by_name[ - _action_name - ] - logger.trace(f"Removed '{_action_name}' instance from cache") - except KeyError: - logger.info( - f"Tried to reload action '{_action_name}', but it was not found" - ) + for handler_name, handler_cache in action_cache.handler_cache_by_name.items(): + if handler_cache.exec_info is not None: + shutdown_action_handler( + action_handler_name=handler_name, + exec_info=handler_cache.exec_info, + ) - if ( - _action_name - in global_state.runner_context.action_handlers_exec_info_by_name - ): - shutdown_action_handler( - action_handler_name=_action_name, - exec_info=global_state.runner_context.action_handlers_exec_info_by_name[ - _action_name - ], - ) + del global_state.runner_context.action_cache_by_name[action_name] + logger.trace(f"Removed '{action_name}' instance from cache") - try: - action_obj = project_def.actions[action_name] - except KeyError: - logger.warning(f"Definition of action {action_name} not found") - continue + try: + action_obj = project_def.actions[action_name] + except KeyError: + logger.warning(f"Definition of action {action_name} not found") + return - action_source = action_obj.source - if action_source is None: - continue - action_package = action_source.split(".")[0] + sources_to_remove = [action_obj.source] + for handler in action_obj.handlers: + sources_to_remove.append(handler.source) + + for source_to_remove in sources_to_remove: + source_package = source_to_remove.split(".")[0] loaded_package_modules = dict( [ (key, value) for key, value in sys.modules.items() - if key.startswith(action_package) + if key.startswith(source_package) and isinstance(value, types.ModuleType) ] ) @@ -147,7 +128,7 @@ def reload_action(action_name: str) -> None: for key in loaded_package_modules: del sys.modules[key] - logger.trace(f"Remove modules of package '{action_package}' from cache") + logger.trace(f"Remove modules of package '{source_package}' from cache") def resolve_package_path(package_name: str) -> str: @@ -162,11 +143,13 @@ def resolve_package_path(package_name: str) -> str: def document_did_open(document_uri: str) -> None: - global_state.runner_context.docs_owned_by_client.append(document_uri) + if global_state.runner_context is not None: + global_state.runner_context.docs_owned_by_client.append(document_uri) def document_did_close(document_uri: str) -> None: - global_state.runner_context.docs_owned_by_client.remove(document_uri) + if global_state.runner_context is not None: + global_state.runner_context.docs_owned_by_client.remove(document_uri) def shutdown_action_handler( @@ -190,19 +173,17 @@ def shutdown_action_handler( def shutdown_all_action_handlers() -> None: logger.trace("Shutdown all action handlers") - for ( - action_handler_name, - exec_info, - ) in global_state.runner_context.action_handlers_exec_info_by_name.items(): - shutdown_action_handler( - action_handler_name=action_handler_name, exec_info=exec_info - ) + for action_cache in global_state.action_cache_by_name.values(): + for handler_name, handler_cache in action_cache.handler_cache_by_name.items(): + if handler_cache.exec_info is not None: + shutdown_action_handler( + action_handler_name=handler_name, exec_info=handler_cache.exec_info + ) def exit_action_handler( action_handler_name: str, exec_info: domain.ActionHandlerExecInfo ) -> None: - # action handler exec info expected to exist in runner_context if ( exec_info.lifecycle is not None and exec_info.lifecycle.on_exit_callable is not None @@ -216,11 +197,11 @@ def exit_action_handler( def exit_all_action_handlers() -> None: logger.trace("Exit all action handlers") - for ( - action_handler_name, - exec_info, - ) in global_state.runner_context.action_handlers_exec_info_by_name.items(): - exit_action_handler( - action_handler_name=action_handler_name, exec_info=exec_info - ) - global_state.runner_context.action_handlers_exec_info_by_name = {} + for action_cache in global_state.action_cache_by_name.values(): + for handler_name, handler_cache in action_cache.handler_cache_by_name.items(): + if handler_cache.exec_info is not None: + exec_info = handler_cache.exec_info + exit_action_handler( + action_handler_name=handler_name, exec_info=exec_info + ) + action_cache.handler_cache_by_name = {} From 14b2bae34219befe11b6b76dd846126abde1c395 Mon Sep 17 00:00:00 2001 From: Vladyslav Hnatiuk Date: Sat, 6 Sep 2025 13:17:50 +0200 Subject: [PATCH 02/11] Move rereading project config and collection actions in `start_runners_with_presets`, because these runners are always started only for this purpose --- src/finecode/cli_app/dump_config.py | 12 -------- src/finecode/cli_app/prepare_envs.py | 13 -------- src/finecode/cli_app/run.py | 12 -------- src/finecode/find_project.py | 23 ++++++++++---- .../lsp_server/endpoints/document_sync.py | 12 ++++---- src/finecode/lsp_server/services.py | 13 -------- src/finecode/proxy_utils.py | 8 ++--- src/finecode/runner/manager.py | 30 +++++++++++++++++++ src/finecode/watch_and_run.py | 2 +- 9 files changed, 59 insertions(+), 66 deletions(-) diff --git a/src/finecode/cli_app/dump_config.py b/src/finecode/cli_app/dump_config.py index 37b0c2a..0e25470 100644 --- a/src/finecode/cli_app/dump_config.py +++ b/src/finecode/cli_app/dump_config.py @@ -61,18 +61,6 @@ async def dump_config(workdir_path: pathlib.Path, project_name: str): f"Starting runners with presets failed: {exception.message}" ) - try: - await read_configs.read_project_config( - project=project, ws_context=ws_context - ) - collect_actions.collect_actions( - project_path=project.dir_path, ws_context=ws_context - ) - except config_models.ConfigurationError as exception: - raise DumpFailed( - f"Rereading project config with presets and collecting actions in {project.dir_path} failed: {exception.message}" - ) - try: await proxy_utils.start_required_environments( actions_by_projects, ws_context diff --git a/src/finecode/cli_app/prepare_envs.py b/src/finecode/cli_app/prepare_envs.py index 1fc8d1e..2ffd73d 100644 --- a/src/finecode/cli_app/prepare_envs.py +++ b/src/finecode/cli_app/prepare_envs.py @@ -84,19 +84,6 @@ async def prepare_envs(workdir_path: pathlib.Path, recreate: bool) -> None: f"Starting runners with presets failed: {exception.message}" ) - for project in projects: - try: - await read_configs.read_project_config( - project=project, ws_context=ws_context - ) - collect_actions.collect_actions( - project_path=project.dir_path, ws_context=ws_context - ) - except config_models.ConfigurationError as exception: - raise PrepareEnvsFailed( - f"Rereading project config with presets and collecting actions in {project.dir_path} failed: {exception.message}" - ) - # now all 'dev_workspace' envs are valid, run 'prepare_runners' in them to create # venvs and install runners and presets in them actions_by_projects: dict[pathlib.Path, list[str]] = { diff --git a/src/finecode/cli_app/run.py b/src/finecode/cli_app/run.py index a19268f..cbd6a0d 100644 --- a/src/finecode/cli_app/run.py +++ b/src/finecode/cli_app/run.py @@ -125,18 +125,6 @@ async def run_actions( logger.error("Unexpected exception:") logger.exception(exception) - # 2. Collect actions in relevant projects - for project in projects: - try: - await read_configs.read_project_config( - project=project, ws_context=ws_context - ) - collect_actions.collect_actions( - project_path=project.dir_path, ws_context=ws_context - ) - except config_models.ConfigurationError as exception: - raise RunFailed(f"Found configuration problem: {exception.message}") - actions_by_projects: dict[pathlib.Path, list[str]] = {} if projects_names is not None: # check that all projects have all actions to detect problem and provide diff --git a/src/finecode/find_project.py b/src/finecode/find_project.py index 3be854a..1f074db 100644 --- a/src/finecode/find_project.py +++ b/src/finecode/find_project.py @@ -4,6 +4,7 @@ from finecode import domain from finecode.context import WorkspaceContext +from finecode.runner import manager as runner_manager class FileNotInWorkspaceException(BaseException): ... @@ -12,7 +13,7 @@ class FileNotInWorkspaceException(BaseException): ... class FileHasNotActionException(BaseException): ... -def find_project_with_action_for_file( +async def find_project_with_action_for_file( file_path: Path, action_name: str, ws_context: WorkspaceContext, @@ -70,10 +71,22 @@ def find_project_with_action_for_file( if project.status == domain.ProjectStatus.NO_FINECODE: continue else: - raise ValueError( - f"Action is related to project {project_dir_path} but its action " - f"cannot be resolved({project.status})" - ) + if project.status == domain.ProjectStatus.CONFIG_VALID: + try: + await runner_manager.get_or_start_runners_with_presets(project_dir_path=project_dir_path, ws_context=ws_context) + except runner_manager.RunnerFailedToStart as exception: + raise ValueError( + f"Action is related to project {project_dir_path} but runner " + f"with presets failed to start in it: {exception.message}" + ) + + assert project.actions is not None + project_actions = project.actions + else: + raise ValueError( + f"Action is related to project {project_dir_path} but its action " + f"cannot be resolved({project.status})" + ) try: next(action for action in project_actions if action.name == action_name) diff --git a/src/finecode/lsp_server/endpoints/document_sync.py b/src/finecode/lsp_server/endpoints/document_sync.py index de67978..613b61e 100644 --- a/src/finecode/lsp_server/endpoints/document_sync.py +++ b/src/finecode/lsp_server/endpoints/document_sync.py @@ -24,7 +24,7 @@ async def document_did_open( projects_paths = [ project_path for project_path, project in global_state.ws_context.ws_projects.items() - if project.status == domain.ProjectStatus.RUNNING + if project.status == domain.ProjectStatus.CONFIG_VALID and file_path.is_relative_to(project_path) ] @@ -34,17 +34,17 @@ async def document_did_open( try: async with asyncio.TaskGroup() as tg: for project_path in projects_paths: - runners_by_env = global_state.ws_context.ws_projects_extension_runners[ - project_path - ] + runners_by_env = global_state.ws_context.ws_projects_extension_runners.get(project_path, {}) for runner in runners_by_env.values(): tg.create_task( runner_client.notify_document_did_open( runner=runner, document_info=document_info ) ) - except ExceptionGroup as e: - logger.error(f"Error while sending opened document: {e}") + except ExceptionGroup as eg: + for exception in eg.exceptions: + logger.exception(exception) + logger.error(f"Error while sending opened document: {eg}") async def document_did_close( diff --git a/src/finecode/lsp_server/services.py b/src/finecode/lsp_server/services.py index c408221..2cfb196 100644 --- a/src/finecode/lsp_server/services.py +++ b/src/finecode/lsp_server/services.py @@ -78,19 +78,6 @@ async def add_workspace_dir( except runner_manager.RunnerFailedToStart as exception: raise ValueError(f"Starting runners with presets failed: {exception.message}") - for project in new_projects: - try: - await read_configs.read_project_config( - project=project, ws_context=global_state.ws_context - ) - collect_actions.collect_actions( - project_path=project.dir_path, ws_context=global_state.ws_context - ) - except config_models.ConfigurationError as exception: - raise ValueError( - f"Rereading project config with presets and collecting actions in {project.dir_path} failed: {exception.message}" - ) - return schemas.AddWorkspaceDirResponse() diff --git a/src/finecode/proxy_utils.py b/src/finecode/proxy_utils.py index a0c9895..db595d6 100644 --- a/src/finecode/proxy_utils.py +++ b/src/finecode/proxy_utils.py @@ -14,11 +14,11 @@ from finecode.services import ActionRunFailed -def find_action_project( +async def find_action_project( file_path: pathlib.Path, action_name: str, ws_context: context.WorkspaceContext ) -> pathlib.Path: try: - project_path = find_project.find_project_with_action_for_file( + project_path = await find_project.find_project_with_action_for_file( file_path=file_path, action_name=action_name, ws_context=ws_context, @@ -51,7 +51,7 @@ async def find_action_project_and_run( params: dict[str, Any], ws_context: context.WorkspaceContext, ) -> runner_client.RunActionResponse: - project_path = find_action_project( + project_path = await find_action_project( file_path=file_path, action_name=action_name, ws_context=ws_context ) project = ws_context.ws_projects[project_path] @@ -227,7 +227,7 @@ async def find_action_project_and_run_with_partial_results( ws_context: context.WorkspaceContext, ) -> collections.abc.AsyncIterator[runner_client.RunActionRawResult]: logger.trace(f"Run {action_name} on {file_path}") - project_path = find_action_project( + project_path = await find_action_project( file_path=file_path, action_name=action_name, ws_context=ws_context ) return run_with_partial_results( diff --git a/src/finecode/runner/manager.py b/src/finecode/runner/manager.py index e6e058a..54d6968 100644 --- a/src/finecode/runner/manager.py +++ b/src/finecode/runner/manager.py @@ -202,6 +202,7 @@ def stop_extension_runner_sync(runner: runner_info.ExtensionRunnerInfo) -> None: async def start_runners_with_presets( projects: list[domain.Project], ws_context: context.WorkspaceContext ) -> None: + # start runners with presets in projects, resolve presets and read project actions new_runners_tasks: list[asyncio.Task] = [] try: # first start runner in 'dev_workspace' env to be able to resolve presets for @@ -236,6 +237,35 @@ async def start_runners_with_presets( raise RunnerFailedToStart( "Failed to initialize runner(s). See previous logs for more details" ) + + for project in projects: + try: + await read_configs.read_project_config( + project=project, ws_context=ws_context + ) + collect_actions.collect_actions( + project_path=project.dir_path, ws_context=ws_context + ) + except config_models.ConfigurationError as exception: + raise RunnerFailedToStart( + f"Reading project config with presets and collecting actions in {project.dir_path} failed: {exception.message}" + ) + + +async def get_or_start_runners_with_presets(project_dir_path: Path, ws_context: context.WorkspaceContext) -> runner_info.ExtensionRunnerInfo: + # project is expected to have status `ProjectStatus.CONFIG_VALID` + has_dev_workspace_runner = 'dev_workspace' in ws_context.ws_projects_extension_runners[project_dir_path] + if not has_dev_workspace_runner: + project = ws_context.ws_projects[project_dir_path] + await start_runners_with_presets([project], ws_context) + dev_workspace_runner = ws_context.ws_projects_extension_runners[project_dir_path]['dev_workspace'] + if dev_workspace_runner.status == runner_info.RunnerStatus.RUNNING: + return dev_workspace_runner + elif dev_workspace_runner.status == runner_info.RunnerStatus.INITIALIZING: + await dev_workspace_runner.initialized_event.wait() + return dev_workspace_runner + else: + raise RunnerFailedToStart(f'Status of dev_workspace runner: {dev_workspace_runner.status}, logs: {dev_workspace_runner.logs_path}') async def start_runner( diff --git a/src/finecode/watch_and_run.py b/src/finecode/watch_and_run.py index a5f7586..1e083db 100644 --- a/src/finecode/watch_and_run.py +++ b/src/finecode/watch_and_run.py @@ -31,7 +31,7 @@ async def watch_and_run( # and lint? for action in ["lint", "format"]: # TODO: this can be cached - project_root = find_project.find_project_with_action_for_file( + project_root = await find_project.find_project_with_action_for_file( file_path=path_to_apply_on, action_name=action, ws_context=ws_context, From dad46cfec9b71c8a8cfbb25fa29155fdefa9bf90 Mon Sep 17 00:00:00 2001 From: Vladyslav Hnatiuk Date: Sat, 6 Sep 2025 13:18:20 +0200 Subject: [PATCH 03/11] Fix parsing of lint messages in ruff and pyrefly handlers --- .../fine_python_pyrefly/fine_python_pyrefly/lint_handler.py | 2 +- extensions/fine_python_ruff/fine_python_ruff/lint_handler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py b/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py index 3e15608..89e8fa5 100644 --- a/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py +++ b/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py @@ -117,7 +117,7 @@ def map_pyrefly_error_to_lint_message(error: dict) -> lint_action.LintMessage: end_column = error['stop_column'] # Determine severity based on error type - error_code = error.get('code', '') + error_code = str(error.get('code', '')) code_description = error.get("name", "") severity = lint_action.LintMessageSeverity.ERROR diff --git a/extensions/fine_python_ruff/fine_python_ruff/lint_handler.py b/extensions/fine_python_ruff/fine_python_ruff/lint_handler.py index 1b138c0..8dd414f 100644 --- a/extensions/fine_python_ruff/fine_python_ruff/lint_handler.py +++ b/extensions/fine_python_ruff/fine_python_ruff/lint_handler.py @@ -135,7 +135,7 @@ def map_ruff_violation_to_lint_message(violation: dict) -> lint_action.LintMessa # Extract line/column info (ruff uses 1-based indexing) start_line = max(1, location.get("row", 1)) start_column = max(0, location.get("column", 0)) - end_line = max(1, end_location.get("row", start_line + 1)) - 1 # Convert to 0-based + end_line = max(1, end_location.get("row", start_line + 1)) end_column = max(0, end_location.get("column", start_column)) # Determine severity based on rule code From 82a666ef19292b76bba349ab17437b3e6c1f679b Mon Sep 17 00:00:00 2001 From: Vladyslav Hnatiuk Date: Thu, 11 Sep 2025 05:55:49 +0200 Subject: [PATCH 04/11] Replace ActionContext by iextensionrunnerinfoprovider. Add file classifier: distinguish source files and tests. Services for getting venv dir, venv site packages. --- .../fine_python_mypy/action.py | 12 ++- .../fine_python_pyrefly/lint_handler.py | 28 +++++- .../fine_python_ruff/format_handler.py | 7 +- .../prepare_envs_install_deps.py | 2 - ...pare_runners_install_runner_and_presets.py | 2 - .../prepare_runners_read_configs.py | 1 - .../src/finecode_extension_api/code_action.py | 8 -- .../iextensionrunnerinfoprovider.py | 12 +++ .../interfaces/iprocessexecutor.py | 3 +- .../interfaces/iprojectfileclassifier.py | 17 ++++ .../interfaces/iprojectinfoprovider.py | 5 ++ .../src/finecode_extension_api/service.py | 3 + .../_services/run_action.py | 28 ++---- .../finecode_extension_runner/di/bootstrap.py | 30 ++++++- .../finecode_extension_runner/di/resolver.py | 15 +++- .../src/finecode_extension_runner/domain.py | 8 +- .../finecode_extension_runner/global_state.py | 2 +- .../impls/extension_runner_info_provider.py | 51 +++++++++++ .../impls/project_file_classifier.py | 87 +++++++++++++++++++ .../impls/project_info_provider.py | 17 ++++ .../finecode_extension_runner/lsp_server.py | 2 + .../src/finecode_extension_runner/schemas.py | 1 + .../src/finecode_extension_runner/services.py | 21 +++-- src/finecode/config/read_configs.py | 2 +- src/finecode/runner/manager.py | 5 +- src/finecode/runner/runner_client.py | 8 +- 26 files changed, 308 insertions(+), 69 deletions(-) create mode 100644 finecode_extension_api/src/finecode_extension_api/interfaces/iextensionrunnerinfoprovider.py create mode 100644 finecode_extension_api/src/finecode_extension_api/interfaces/iprojectfileclassifier.py create mode 100644 finecode_extension_api/src/finecode_extension_api/service.py create mode 100644 finecode_extension_runner/src/finecode_extension_runner/impls/extension_runner_info_provider.py create mode 100644 finecode_extension_runner/src/finecode_extension_runner/impls/project_file_classifier.py diff --git a/extensions/fine_python_mypy/fine_python_mypy/action.py b/extensions/fine_python_mypy/fine_python_mypy/action.py index 790241e..c52adf5 100644 --- a/extensions/fine_python_mypy/fine_python_mypy/action.py +++ b/extensions/fine_python_mypy/fine_python_mypy/action.py @@ -15,6 +15,8 @@ icommandrunner, ifilemanager, ilogger, + iextensionrunnerinfoprovider, + iprojectinfoprovider ) @@ -44,14 +46,16 @@ class MypyLintHandler( def __init__( self, - context: code_action.ActionContext, + extension_runner_info_provider: iextensionrunnerinfoprovider.IExtensionRunnerInfoProvider, + project_info_provider: iprojectinfoprovider.IProjectInfoProvider, cache: icache.ICache, logger: ilogger.ILogger, file_manager: ifilemanager.IFileManager, lifecycle: code_action.ActionHandlerLifecycle, command_runner: icommandrunner.ICommandRunner, ) -> None: - self.context = context + self.extension_runner_info_provider = extension_runner_info_provider + self.project_info_provider = project_info_provider self.cache = cache self.logger = logger self.file_manager = file_manager @@ -189,7 +193,7 @@ async def run( file_paths = [file_path async for file_path in payload] files_by_projects: dict[Path, list[Path]] = self.group_files_by_projects( - file_paths, self.context.project_dir + file_paths, self.project_info_provider.get_current_project_dir_path() ) for project_path, project_files in files_by_projects.items(): @@ -225,7 +229,7 @@ def exit(self) -> None: self.logger.error(str(error)) def _get_status_file_path(self, dmypy_cwd: Path) -> Path: - file_dir_path = self.context.cache_dir + file_dir_path = self.extension_runner_info_provider.get_cache_dir_path() # use hash to avoid name conflict if python packages have the same name file_dir_path_hash = hashlib.md5(str(dmypy_cwd).encode("utf-8")).hexdigest() file_path = ( diff --git a/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py b/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py index 89e8fa5..7dbb89c 100644 --- a/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py +++ b/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py @@ -7,7 +7,7 @@ from finecode_extension_api import code_action from finecode_extension_api.actions import lint as lint_action -from finecode_extension_api.interfaces import icache, icommandrunner, ilogger, ifilemanager +from finecode_extension_api.interfaces import icache, icommandrunner, ilogger, ifilemanager, iprojectfileclassifier, iextensionrunnerinfoprovider @dataclasses.dataclass @@ -32,12 +32,16 @@ def __init__( logger: ilogger.ILogger, file_manager: ifilemanager.IFileManager, command_runner: icommandrunner.ICommandRunner, + project_file_classifier: iprojectfileclassifier.IProjectFileClassifier, + extension_runner_info_provider: iextensionrunnerinfoprovider.IExtensionRunnerInfoProvider ) -> None: self.config = config self.cache = cache self.logger = logger self.file_manager = file_manager self.command_runner = command_runner + self.project_file_classifier = project_file_classifier + self.extension_runner_info_provider = extension_runner_info_provider self.pyrefly_bin_path = Path(sys.executable).parent / "pyrefly" @@ -82,15 +86,31 @@ async def run_pyrefly_lint_on_single_file( ) -> list[lint_action.LintMessage]: """Run pyrefly type checking on a single file""" lint_messages: list[lint_action.LintMessage] = [] + + try: + # project file classifier caches result, we can just get it each time again + file_type = self.project_file_classifier.get_project_file_type(file_path=file_path) + file_env = self.project_file_classifier.get_env_for_file_type(file_type=file_type) + except NotImplementedError: + self.logger.warning(f"Skip {file_path} because file type or env for it could be determined") + return lint_messages + + venv_dir_path = self.extension_runner_info_provider.get_venv_dir_path_of_env(env_name=file_env) + site_package_pathes = self.extension_runner_info_provider.get_venv_site_packages(venv_dir_path=venv_dir_path) cmd = [ str(self.pyrefly_bin_path), "check", - "--output-format", - "json", - str(file_path), + "--output-format=json", + "--disable-search-path-heuristics=true", + "--skip-interpreter-query", + "--python-version='3.11'" # TODO ] + for path in site_package_pathes: + cmd.append(f'--site-package-path={str(path)}') + cmd.append(str(file_path)) + cmd_str = " ".join(cmd) pyrefly_process = await self.command_runner.run(cmd_str) diff --git a/extensions/fine_python_ruff/fine_python_ruff/format_handler.py b/extensions/fine_python_ruff/fine_python_ruff/format_handler.py index f8a82d0..2bd0bab 100644 --- a/extensions/fine_python_ruff/fine_python_ruff/format_handler.py +++ b/extensions/fine_python_ruff/fine_python_ruff/format_handler.py @@ -11,7 +11,7 @@ from finecode_extension_api import code_action from finecode_extension_api.actions import format as format_action -from finecode_extension_api.interfaces import icache, icommandrunner, ilogger +from finecode_extension_api.interfaces import icache, icommandrunner, ilogger, iextensionrunnerinfoprovider @dataclasses.dataclass @@ -31,7 +31,7 @@ class RuffFormatHandler( def __init__( self, config: RuffFormatHandlerConfig, - context: code_action.ActionContext, + extension_runner_info_provider: iextensionrunnerinfoprovider.IExtensionRunnerInfoProvider, logger: ilogger.ILogger, cache: icache.ICache, command_runner: icommandrunner.ICommandRunner, @@ -41,6 +41,7 @@ def __init__( self.logger = logger self.cache = cache self.command_runner = command_runner + self.extension_runner_info_provider = extension_runner_info_provider self.ruff_bin_path = Path(sys.executable).parent / "ruff" @@ -89,7 +90,7 @@ async def format_one(self, file_path: Path, file_content: str) -> tuple[str, boo str(self.ruff_bin_path), "format", "--cache-dir", - str(self.context.cache_dir / ".ruff_cache"), + str(self.extension_runner_info_provider.get_cache_dir_path() / ".ruff_cache"), "--line-length", str(self.config.line_length), f'--config="indent-width={str(self.config.indent_width)}"', diff --git a/finecode_builtin_handlers/src/finecode_builtin_handlers/prepare_envs_install_deps.py b/finecode_builtin_handlers/src/finecode_builtin_handlers/prepare_envs_install_deps.py index b8c4c95..e7c46f4 100644 --- a/finecode_builtin_handlers/src/finecode_builtin_handlers/prepare_envs_install_deps.py +++ b/finecode_builtin_handlers/src/finecode_builtin_handlers/prepare_envs_install_deps.py @@ -1,14 +1,12 @@ import asyncio import dataclasses import itertools -import shutil from finecode_extension_api import code_action from finecode_extension_api.actions import prepare_envs as prepare_envs_action from finecode_extension_api.interfaces import ( iactionrunner, ilogger, - iprojectinfoprovider, ) from finecode_builtin_handlers import dependency_config_utils diff --git a/finecode_builtin_handlers/src/finecode_builtin_handlers/prepare_runners_install_runner_and_presets.py b/finecode_builtin_handlers/src/finecode_builtin_handlers/prepare_runners_install_runner_and_presets.py index c7493ff..a897be0 100644 --- a/finecode_builtin_handlers/src/finecode_builtin_handlers/prepare_runners_install_runner_and_presets.py +++ b/finecode_builtin_handlers/src/finecode_builtin_handlers/prepare_runners_install_runner_and_presets.py @@ -1,7 +1,6 @@ import asyncio import dataclasses import itertools -import shutil import typing from finecode_extension_api import code_action @@ -9,7 +8,6 @@ from finecode_extension_api.interfaces import ( iactionrunner, ilogger, - iprojectinfoprovider, ) from finecode_builtin_handlers import dependency_config_utils diff --git a/finecode_builtin_handlers/src/finecode_builtin_handlers/prepare_runners_read_configs.py b/finecode_builtin_handlers/src/finecode_builtin_handlers/prepare_runners_read_configs.py index 915ecc2..7d77448 100644 --- a/finecode_builtin_handlers/src/finecode_builtin_handlers/prepare_runners_read_configs.py +++ b/finecode_builtin_handlers/src/finecode_builtin_handlers/prepare_runners_read_configs.py @@ -1,7 +1,6 @@ import asyncio import dataclasses import pathlib -import shutil import typing from finecode_extension_api import code_action diff --git a/finecode_extension_api/src/finecode_extension_api/code_action.py b/finecode_extension_api/src/finecode_extension_api/code_action.py index e974628..cde49bf 100644 --- a/finecode_extension_api/src/finecode_extension_api/code_action.py +++ b/finecode_extension_api/src/finecode_extension_api/code_action.py @@ -5,7 +5,6 @@ import dataclasses import enum import typing -from pathlib import Path from typing import Generic, Protocol, TypeVar from finecode_extension_api import partialresultscheduler, textstyler @@ -73,13 +72,6 @@ def __init__(self, run_id: int) -> None: self.partial_result_scheduler = partialresultscheduler.PartialResultScheduler() -class ActionContext: - def __init__(self, project_dir: Path, cache_dir: Path) -> None: - self.project_dir = project_dir - # runner-specific cache dir - self.cache_dir = cache_dir - - @dataclasses.dataclass class ActionConfig: run_handlers_concurrently: bool = False diff --git a/finecode_extension_api/src/finecode_extension_api/interfaces/iextensionrunnerinfoprovider.py b/finecode_extension_api/src/finecode_extension_api/interfaces/iextensionrunnerinfoprovider.py new file mode 100644 index 0000000..280c76f --- /dev/null +++ b/finecode_extension_api/src/finecode_extension_api/interfaces/iextensionrunnerinfoprovider.py @@ -0,0 +1,12 @@ +import pathlib +from typing import Protocol + + +class IExtensionRunnerInfoProvider(Protocol): + def get_cache_dir_path(self) -> pathlib.Path: ... + + def get_venv_dir_path_of_env(self, env_name: str) -> pathlib.Path: + ... + + def get_venv_site_packages(self, venv_dir_path: pathlib.Path) -> list[pathlib.Path]: + ... \ No newline at end of file diff --git a/finecode_extension_api/src/finecode_extension_api/interfaces/iprocessexecutor.py b/finecode_extension_api/src/finecode_extension_api/interfaces/iprocessexecutor.py index ebb1cda..a4fd973 100644 --- a/finecode_extension_api/src/finecode_extension_api/interfaces/iprocessexecutor.py +++ b/finecode_extension_api/src/finecode_extension_api/interfaces/iprocessexecutor.py @@ -1,11 +1,10 @@ import typing -from asyncio import BaseProtocol T = typing.TypeVar("T") P = typing.ParamSpec("P") -class IProcessExecutor(BaseProtocol): +class IProcessExecutor(typing.Protocol): async def submit( self, func: typing.Callable[P, T], *args: P.args, **kwargs: P.kwargs ): ... diff --git a/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectfileclassifier.py b/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectfileclassifier.py new file mode 100644 index 0000000..dd15b7a --- /dev/null +++ b/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectfileclassifier.py @@ -0,0 +1,17 @@ +import enum +import typing +import pathlib + + +class ProjectFileType(enum.Enum): + SOURCE = enum.auto() + TEST = enum.auto() + UNKNOWN = enum.auto() + + +class IProjectFileClassifier(typing.Protocol): + def get_project_file_type(self, file_path: pathlib.Path) -> ProjectFileType: + ... + + def get_env_for_file_type(self, file_type: ProjectFileType) -> str: + ... diff --git a/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectinfoprovider.py b/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectinfoprovider.py index 6e3af54..02f1243 100644 --- a/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectinfoprovider.py +++ b/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectinfoprovider.py @@ -3,8 +3,13 @@ class IProjectInfoProvider(Protocol): + def get_current_project_dir_path(self) -> pathlib.Path: ... + def get_current_project_def_path(self) -> pathlib.Path: ... + async def get_current_project_package_name(self) -> str: + ... + async def get_project_raw_config( self, project_def_path: pathlib.Path ) -> dict[str, Any]: ... diff --git a/finecode_extension_api/src/finecode_extension_api/service.py b/finecode_extension_api/src/finecode_extension_api/service.py new file mode 100644 index 0000000..75592ce --- /dev/null +++ b/finecode_extension_api/src/finecode_extension_api/service.py @@ -0,0 +1,3 @@ +class Service: + async def init(self) -> None: + ... diff --git a/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py b/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py index 3456378..fc2d82b 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py +++ b/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py @@ -101,7 +101,7 @@ async def run_action( run_context: code_action.RunActionContext | None = None if action_exec_info.run_context_type is not None: - constructor_args = resolve_func_args_with_di( + constructor_args = await resolve_func_args_with_di( action_exec_info.run_context_type.__init__, known_args={"run_id": lambda _: run_id}, params_to_ignore=["self"], @@ -114,16 +114,6 @@ async def run_action( action_result: code_action.RunActionResult | None = None runner_context = global_state.runner_context - # instantiate only on demand? - project_path = project_def.path - project_cache_dir = project_path / ".venvs" / global_state.env_name / "cache" - if not project_cache_dir.exists(): - project_cache_dir.mkdir() - - action_context = code_action.ActionContext( - project_dir=project_path, cache_dir=project_cache_dir - ) - # TODO: take value from action config execute_handlers_concurrently = action.name == "lint" partial_result_token = options.partial_result_token @@ -144,7 +134,6 @@ async def run_action( run_context=run_context, run_id=run_id, action_cache=action_cache, - action_context=action_context, action_exec_info=action_exec_info, runner_context=runner_context, ) @@ -228,7 +217,6 @@ async def run_action( run_context=run_context, run_id=run_id, action_cache=action_cache, - action_context=action_context, action_exec_info=action_exec_info, runner_context=runner_context, ) @@ -259,7 +247,6 @@ async def run_action( run_context=run_context, run_id=run_id, action_cache=action_cache, - action_context=action_context, action_exec_info=action_exec_info, runner_context=runner_context, ) @@ -349,7 +336,7 @@ def create_action_exec_info(action: domain.Action) -> domain.ActionExecInfo: return action_exec_info -def resolve_func_args_with_di( +async def resolve_func_args_with_di( func: typing.Callable, known_args: dict[str, typing.Callable[[typing.Any], typing.Any]] | None = None, params_to_ignore: list[str] | None = None, @@ -373,7 +360,7 @@ def resolve_func_args_with_di( else: # TODO: handle errors param_type = func_annotations[param_name] - param_value = di_resolver.get_service_instance(param_type) + param_value = await di_resolver.get_service_instance(param_type) args[param_name] = param_value return args @@ -386,7 +373,6 @@ async def execute_action_handler( run_id: int, action_exec_info: domain.ActionExecInfo, action_cache: domain.ActionCache, - action_context: code_action.ActionContext, runner_context: context.RunnerContext, ) -> code_action.RunActionResult: logger.trace(f"R{run_id} | Run {handler.name} on {str(payload)[:100]}...") @@ -439,9 +425,6 @@ def get_handler_config(param_type): # TODO: validation errors return param_type(**handler_raw_config) - def get_action_context(param_type): - return action_context - def get_process_executor(param_type): return action_exec_info.process_executor @@ -450,11 +433,10 @@ def get_process_executor(param_type): # is interrupted by stopping ER handler_cache.exec_info = exec_info if inspect.isclass(action_handler): - args = resolve_func_args_with_di( + args = await resolve_func_args_with_di( func=action_handler.__init__, known_args={ "config": get_handler_config, - "context": get_action_context, "process_executor": get_process_executor, }, params_to_ignore=["self"], @@ -499,7 +481,7 @@ def get_run_context(param_type): # DI in `run` function is allowed only for action handlers in form of functions. # `run` in classes may not have additional parameters, constructor parameters should # be used instead. TODO: Validate? - args = resolve_func_args_with_di( + args = await resolve_func_args_with_di( func=handler_run_func, known_args={"payload": get_run_payload, "run_context": get_run_context}, ) diff --git a/finecode_extension_runner/src/finecode_extension_runner/di/bootstrap.py b/finecode_extension_runner/src/finecode_extension_runner/di/bootstrap.py index 6299bb6..50bd24f 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/di/bootstrap.py +++ b/finecode_extension_runner/src/finecode_extension_runner/di/bootstrap.py @@ -19,10 +19,12 @@ ifilemanager, ilogger, iprojectinfoprovider, + iextensionrunnerinfoprovider, + iprojectfileclassifier ) from finecode_extension_runner import global_state, schemas from finecode_extension_runner._services import run_action -from finecode_extension_runner.di import _state +from finecode_extension_runner.di import _state, resolver from finecode_extension_runner.impls import ( action_runner, command_runner, @@ -30,6 +32,8 @@ inmemory_cache, loguru_logger, project_info_provider, + extension_runner_info_provider, + project_file_classifier ) @@ -38,6 +42,7 @@ def bootstrap( save_document_func: Callable, project_def_path_getter: Callable[[], pathlib.Path], project_raw_config_getter: Callable[[str], Awaitable[dict[str, Any]]], + cache_dir_path_getter: Callable[[], pathlib.Path], ): # logger_instance = loguru_logger.LoguruLogger() logger_instance = loguru_logger.get_logger() @@ -73,6 +78,11 @@ def bootstrap( project_def_path_getter=project_def_path_getter, project_raw_config_getter=project_raw_config_getter, ) + _state.factories[iextensionrunnerinfoprovider.IExtensionRunnerInfoProvider] = functools.partial( + extension_runner_info_provider_factory, + cache_dir_path_getter=cache_dir_path_getter + ) + _state.factories[iprojectfileclassifier.IProjectFileClassifier] = project_file_classifier_factory # TODO: parameters from config @@ -116,3 +126,21 @@ def project_info_provider_factory( project_def_path_getter=project_def_path_getter, project_raw_config_getter=project_raw_config_getter, ) + + +async def extension_runner_info_provider_factory( + container, + cache_dir_path_getter: Callable[[], pathlib.Path], +): + logger = await resolver.get_service_instance(ilogger.ILogger) + return extension_runner_info_provider.ExtensionRunnerInfoProvider( + cache_dir_path_getter=cache_dir_path_getter, + logger=logger + ) + + +async def project_file_classifier_factory( + container, +): + project_info_provider = await resolver.get_service_instance(iprojectinfoprovider.IProjectInfoProvider) + return project_file_classifier.ProjectFileClassifier(project_info_provider=project_info_provider) diff --git a/finecode_extension_runner/src/finecode_extension_runner/di/resolver.py b/finecode_extension_runner/src/finecode_extension_runner/di/resolver.py index a3f11c9..a1cf725 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/di/resolver.py +++ b/finecode_extension_runner/src/finecode_extension_runner/di/resolver.py @@ -1,13 +1,14 @@ from typing import Type, TypeVar +import inspect -from finecode_extension_api import code_action +from finecode_extension_api import code_action, service from ._state import container, factories T = TypeVar("T") -def get_service_instance(service_type: Type[T]) -> T: +async def get_service_instance(service_type: Type[T]) -> T: if service_type == code_action.ActionHandlerLifecycle: return code_action.ActionHandlerLifecycle() @@ -16,9 +17,17 @@ def get_service_instance(service_type: Type[T]) -> T: return container[service_type] else: if service_type in factories: - service_instance = factories[service_type](container) + factory_result = factories[service_type](container) else: raise ValueError(f"No implementation found for {service_type}") + if inspect.isawaitable(factory_result): + service_instance = await factory_result + else: + service_instance = factory_result + + if isinstance(service_instance, service.Service): + await service_instance.init() + container[service_type] = service_instance return service_instance diff --git a/finecode_extension_runner/src/finecode_extension_runner/domain.py b/finecode_extension_runner/src/finecode_extension_runner/domain.py index 4f91569..c3a64a4 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/domain.py +++ b/finecode_extension_runner/src/finecode_extension_runner/domain.py @@ -34,17 +34,19 @@ class Project: def __init__( self, name: str, - path: Path, + dir_path: Path, + def_path: Path, actions: dict[str, Action], action_handler_configs: dict[str, dict[str, typing.Any]], ) -> None: self.name = name - self.path = path + self.dir_path = dir_path + self.def_path = def_path self.actions = actions self.action_handler_configs = action_handler_configs def __str__(self) -> str: - return f'Project(name="{self.name}", path="{self.path}")' + return f'Project(name="{self.name}", dir_path="{self.dir_path}")' class ActionExecInfo: diff --git a/finecode_extension_runner/src/finecode_extension_runner/global_state.py b/finecode_extension_runner/src/finecode_extension_runner/global_state.py index 879c5f3..b5ba424 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/global_state.py +++ b/finecode_extension_runner/src/finecode_extension_runner/global_state.py @@ -4,7 +4,7 @@ import finecode_extension_runner.context as context runner_context: context.RunnerContext | None = None -# it's the same as `runner_context.project.path`, but it's available from the start of +# it's the same as `runner_context.project.dir_path`, but it's available from the start of # the runner, not from updating the config project_dir_path: Path | None = None log_level: Literal["TRACE", "INFO"] = "INFO" diff --git a/finecode_extension_runner/src/finecode_extension_runner/impls/extension_runner_info_provider.py b/finecode_extension_runner/src/finecode_extension_runner/impls/extension_runner_info_provider.py new file mode 100644 index 0000000..d379a62 --- /dev/null +++ b/finecode_extension_runner/src/finecode_extension_runner/impls/extension_runner_info_provider.py @@ -0,0 +1,51 @@ +import pathlib +from typing import Callable + +from finecode_extension_api.interfaces import iextensionrunnerinfoprovider, ilogger + + +class ExtensionRunnerInfoProvider(iextensionrunnerinfoprovider.IExtensionRunnerInfoProvider): + def __init__( + self, + cache_dir_path_getter: Callable[[], pathlib.Path], + logger: ilogger.ILogger + ) -> None: + self.cache_dir_path_getter = cache_dir_path_getter + self.logger = logger + + self._site_packages_cache: dict[pathlib.Path, list[pathlib.Path]] = {} + + def get_cache_dir_path(self) -> pathlib.Path: + return self.cache_dir_path_getter() + + def get_venv_dir_path_of_env(self, env_name: str) -> pathlib.Path: + cache_dir_path = self.get_cache_dir_path() + # assume cache dir is directly in venv + current_venv_dir_path = cache_dir_path.parent + venvs_dir_path = current_venv_dir_path.parent + return venvs_dir_path / env_name + + def get_venv_site_packages(self, venv_dir_path: pathlib.Path) -> list[pathlib.Path]: + # venv site packages can be cached because they don't change and if user runs + # prepare-envs or updates environment in any other way, current ER should be + # reloaded and cache will be automatically cleared + if venv_dir_path in self._site_packages_cache: + return self._site_packages_cache[venv_dir_path] + + site_packages: list[pathlib.Path] = [] + for lib_dir_name in ['lib', 'lib64']: + lib_dir_path = venv_dir_path / lib_dir_name + + if not lib_dir_path.exists(): + continue + + # assume there is only one python version in venv + lib_python_dir_path = next(dir_path for dir_path in lib_dir_path.iterdir() if dir_path.is_dir() and dir_path.name.startswith('python')) + site_packages_path = lib_python_dir_path / 'site-packages' + if site_packages_path.exists(): + site_packages.append(site_packages_path) + else: + self.logger.warning(f"site-packages directory expected in {lib_python_dir_path}, but wasn't exist. Venv seems to be invalid") + + self._site_packages_cache[venv_dir_path] = site_packages + return site_packages diff --git a/finecode_extension_runner/src/finecode_extension_runner/impls/project_file_classifier.py b/finecode_extension_runner/src/finecode_extension_runner/impls/project_file_classifier.py new file mode 100644 index 0000000..64097cc --- /dev/null +++ b/finecode_extension_runner/src/finecode_extension_runner/impls/project_file_classifier.py @@ -0,0 +1,87 @@ +import enum +import pathlib + +from finecode_extension_api.interfaces import iprojectfileclassifier, iprojectinfoprovider +from finecode_extension_api import service + + +class ProjectLayout(enum.Enum): + SRC = enum.auto() + FLAT = enum.auto() + CUSTOM = enum.auto() + + +class ProjectFileClassifier(iprojectfileclassifier.IProjectFileClassifier, service.Service): + # requirements: + # - all project sources should be in a single directory + # - if tests are outside of sources, they should be in a single directory + + def __init__(self, project_info_provider: iprojectinfoprovider.IProjectInfoProvider) -> None: + self.project_info_provider = project_info_provider + # ProjectFileClassifier is instantiated as singletone, cache can be stored in + # object + self._file_type_by_path: dict[pathlib.Path, iprojectfileclassifier.ProjectFileType] = {} + + self.project_layout: ProjectLayout + self.project_src_dir_path: pathlib.Path + self.project_tests_dir_path: pathlib.Path + + async def init(self) -> None: + project_dir_path = self.project_info_provider.get_current_project_dir_path() + project_package_name = await self.project_info_provider.get_current_project_package_name() + self.project_layout = self._get_project_layout(project_dir_path, project_package_name) + if self.project_layout == ProjectLayout.SRC: + self.project_src_dir_path = project_dir_path / 'src' + elif self.project_layout == ProjectLayout.FLAT: + self.project_src_dir_path = project_dir_path / project_package_name + else: + self.project_src_dir_path = None + + self.project_tests_dir_path: pathlib.Path = project_dir_path / 'tests' + + def _get_project_layout(self, project_dir_path: pathlib.Path, project_package_name: str) -> ProjectLayout: + if (project_dir_path / 'src').exists(): + return ProjectLayout.SRC + elif (project_dir_path / project_package_name).exists(): + return ProjectLayout.FLAT + else: + return ProjectLayout.CUSTOM + + def get_project_file_type(self, file_path: pathlib.Path) -> iprojectfileclassifier.ProjectFileType: + if self.project_src_dir_path is None: + raise NotImplementedError(f'{self.project_layout} project layout is not supported') + + if file_path in self._file_type_by_path: + # return cached value if exist + return self._file_type_by_path[file_path] + + if file_path.is_relative_to(self.project_src_dir_path): + file_path_relative_to_project = file_path.relative_to(self.project_src_dir_path) + if '__tests__' in file_path_relative_to_project.parts or 'tests' in file_path_relative_to_project.parts: + file_type = iprojectfileclassifier.ProjectFileType.TEST + else: + file_type = iprojectfileclassifier.ProjectFileType.SOURCE + else: + # not source, check whether test + if file_path.is_relative_to(self.project_tests_dir_path): + file_type = iprojectfileclassifier.ProjectFileType.TEST + else: + file_type = iprojectfileclassifier.ProjectFileType.UNKNOWN + + # cache + self._file_type_by_path[file_path] = file_type + + return file_type + + def get_env_for_file_type(self, file_type: iprojectfileclassifier.ProjectFileType) -> str: + match file_type: + case iprojectfileclassifier.ProjectFileType.SOURCE: + return 'runtime' + case iprojectfileclassifier.ProjectFileType.TEST: + # TODO: dynamic. In future test tool can be installed in any env, we + # need a way to define it in config and get it here + # TODO: there can be also e2e tests that don't use runtime and are in + # e.g. dev_no_runtime env + return 'dev' + case _: + raise NotImplementedError("") diff --git a/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py b/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py index b7eee0c..1a38dc2 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py +++ b/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py @@ -4,6 +4,11 @@ from finecode_extension_api.interfaces import iprojectinfoprovider +class InvalidProjectConfig(Exception): + def __init__(self, message: str) -> None: + self.message = message + + class ProjectInfoProvider(iprojectinfoprovider.IProjectInfoProvider): def __init__( self, @@ -13,9 +18,21 @@ def __init__( self.project_def_path_getter = project_def_path_getter self.project_raw_config_getter = project_raw_config_getter + def get_current_project_dir_path(self) -> pathlib.Path: + project_def_path = self.project_def_path_getter() + return project_def_path.parent + def get_current_project_def_path(self) -> pathlib.Path: return self.project_def_path_getter() + async def get_current_project_package_name(self) -> str: + project_raw_config = await self.get_current_project_raw_config() + raw_name = project_raw_config.get('project', {}).get('name', None) + if raw_name is None: + raise InvalidProjectConfig("project.name not found in project config") + + return raw_name.replace('-', '_') + async def get_project_raw_config( self, project_def_path: pathlib.Path ) -> dict[str, Any]: diff --git a/finecode_extension_runner/src/finecode_extension_runner/lsp_server.py b/finecode_extension_runner/src/finecode_extension_runner/lsp_server.py index 938d345..27b10b3 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/lsp_server.py +++ b/finecode_extension_runner/src/finecode_extension_runner/lsp_server.py @@ -169,6 +169,7 @@ async def update_config( ls: lsp_server.LanguageServer, working_dir: pathlib.Path, project_name: str, + project_def_path: pathlib.Path, config: dict[str, typing.Any], ): logger.trace(f"Update config: {working_dir} {project_name} {config}") @@ -179,6 +180,7 @@ async def update_config( request = schemas.UpdateConfigRequest( working_dir=working_dir, project_name=project_name, + project_def_path=project_def_path, actions={ action["name"]: schemas.Action( name=action["name"], diff --git a/finecode_extension_runner/src/finecode_extension_runner/schemas.py b/finecode_extension_runner/src/finecode_extension_runner/schemas.py index b8a1ff9..6687897 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/schemas.py +++ b/finecode_extension_runner/src/finecode_extension_runner/schemas.py @@ -28,6 +28,7 @@ class Action(BaseSchema): class UpdateConfigRequest(BaseSchema): working_dir: Path project_name: str + project_def_path: Path actions: dict[str, Action] action_handler_configs: dict[str, dict[str, Any]] diff --git a/finecode_extension_runner/src/finecode_extension_runner/services.py b/finecode_extension_runner/src/finecode_extension_runner/services.py index 5a695b0..4cbddad 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/services.py +++ b/finecode_extension_runner/src/finecode_extension_runner/services.py @@ -23,7 +23,7 @@ async def update_config( [str], typing.Awaitable[dict[str, typing.Any]] ], ) -> schemas.UpdateConfigResponse: - project_path = Path(request.working_dir) + project_dir_path = Path(request.working_dir) actions: dict[str, domain.Action] = {} for action_name, action_schema_obj in request.actions.items(): @@ -47,7 +47,8 @@ async def update_config( global_state.runner_context = context.RunnerContext( project=domain.Project( name=request.project_name, - path=project_path, + dir_path=project_dir_path, + def_path=request.project_def_path, actions=actions, action_handler_configs=request.action_handler_configs, ), @@ -57,13 +58,23 @@ async def update_config( # bootstrap here. Should be changed after adding updating configuration on the fly. def project_def_path_getter() -> Path: assert global_state.runner_context is not None - return global_state.runner_context.project.path + return global_state.runner_context.project.def_path + + def cache_dir_path_getter() -> Path: + assert global_state.runner_context is not None + project_dir_path = global_state.runner_context.project.dir_path + project_cache_dir = project_dir_path / ".venvs" / global_state.env_name / "cache" + if not project_cache_dir.exists(): + project_cache_dir.mkdir() + + return project_cache_dir di_bootstrap.bootstrap( get_document_func=document_requester, save_document_func=document_saver, project_def_path_getter=project_def_path_getter, project_raw_config_getter=project_raw_config_getter, + cache_dir_path_getter=cache_dir_path_getter ) return schemas.UpdateConfigResponse() @@ -173,7 +184,7 @@ def shutdown_action_handler( def shutdown_all_action_handlers() -> None: logger.trace("Shutdown all action handlers") - for action_cache in global_state.action_cache_by_name.values(): + for action_cache in global_state.runner_context.action_cache_by_name.values(): for handler_name, handler_cache in action_cache.handler_cache_by_name.items(): if handler_cache.exec_info is not None: shutdown_action_handler( @@ -197,7 +208,7 @@ def exit_action_handler( def exit_all_action_handlers() -> None: logger.trace("Exit all action handlers") - for action_cache in global_state.action_cache_by_name.values(): + for action_cache in global_state.runner_context.action_cache_by_name.values(): for handler_name, handler_cache in action_cache.handler_cache_by_name.items(): if handler_cache.exec_info is not None: exec_info = handler_cache.exec_info diff --git a/src/finecode/config/read_configs.py b/src/finecode/config/read_configs.py index 9806ffb..7501943 100644 --- a/src/finecode/config/read_configs.py +++ b/src/finecode/config/read_configs.py @@ -59,7 +59,7 @@ async def read_projects_in_dir( actions=actions, env_configs={}, ) - is_new_project = not def_file.parent in ws_context.ws_projects + is_new_project = def_file.parent not in ws_context.ws_projects ws_context.ws_projects[def_file.parent] = new_project if is_new_project: new_projects.append(new_project) diff --git a/src/finecode/runner/manager.py b/src/finecode/runner/manager.py index 54d6968..117110e 100644 --- a/src/finecode/runner/manager.py +++ b/src/finecode/runner/manager.py @@ -141,6 +141,7 @@ async def on_progress(params: types.ProgressParams): register_progress_feature(on_progress) async def get_project_raw_config(params): + logger.debug(f"Get project raw config: {params}") project_def_path_str = params.projectDefPath project_def_path = Path(project_def_path_str) try: @@ -379,7 +380,7 @@ async def _init_lsp_client( f"Runner failed to notify about initialization: {error}" ) - logger.debug("LSP Client initialized") + logger.debug(f"LSP Client for initialized: {runner.readable_id}") async def update_runner_config( @@ -390,7 +391,7 @@ async def update_runner_config( actions=project.actions, action_handler_configs=project.action_handler_configs ) try: - await runner_client.update_config(runner, config) + await runner_client.update_config(runner, project.def_path, config) except runner_client.BaseRunnerRequestException as exception: runner.status = runner_info.RunnerStatus.FAILED await notify_project_changed(project) diff --git a/src/finecode/runner/runner_client.py b/src/finecode/runner/runner_client.py index 61c697d..575e386 100644 --- a/src/finecode/runner/runner_client.py +++ b/src/finecode/runner/runner_client.py @@ -9,6 +9,7 @@ import enum import json import typing +import pathlib from typing import TYPE_CHECKING, Any from loguru import logger @@ -73,9 +74,6 @@ async def send_request( f" no response on {method}" ) except TimeoutError: - # can this happen? - # if runner.client._server.returncode != None: - # await log_process_log_streams(process=runner.client._server) raise ResponseTimeout( f"Timeout {timeout}s for response on {method} to" f" runner {runner.readable_id}" @@ -141,6 +139,7 @@ async def initialize( client_info=types.ClientInfo(name=client_name, version=client_version), trace=types.TraceValue.Verbose, ), + timeout=20 ) @@ -249,7 +248,7 @@ class RunnerConfig: action_handler_configs: dict[str, dict[str, Any]] -async def update_config(runner: ExtensionRunnerInfo, config: RunnerConfig) -> None: +async def update_config(runner: ExtensionRunnerInfo, project_def_path: pathlib.Path, config: RunnerConfig) -> None: await send_request( runner=runner, method=types.WORKSPACE_EXECUTE_COMMAND, @@ -258,6 +257,7 @@ async def update_config(runner: ExtensionRunnerInfo, config: RunnerConfig) -> No arguments=[ runner.working_dir_path.as_posix(), runner.working_dir_path.stem, + project_def_path.as_posix(), config, ], ), From 7a23ce6ad70609164ffdf3c92172c6d75d0782e0 Mon Sep 17 00:00:00 2001 From: Vladyslav Hnatiuk Date: Sun, 14 Sep 2025 08:42:48 +0200 Subject: [PATCH 05/11] Find python files in project using new list_project_files_by_lang action instead of rglobbing. Fix repeating lint messages in IDE (cached result of first handler was modified without copying). Restructure WM services to avoid cycle imports. --- extensions/fine_python_package_info/README.md | 0 .../fine_python_package_info/__init__.py | 7 + .../list_project_files_by_lang_python.py | 48 +++ .../py_package_layout_info_provider.py | 47 ++ .../fine_python_package_info/pyproject.toml | 8 + extensions/fine_python_package_info/setup.py | 67 +++ .../tests/__init__.py | 1 + extensions/fine_python_pip/setup.py | 1 + .../fine_python_ruff/format_handler.py | 1 - .../src/finecode_builtin_handlers/__init__.py | 2 +- .../actions/list_project_files_by_lang.py | 53 +++ .../actions/prepare_envs.py | 6 +- .../actions/prepare_runners.py | 6 +- .../iextensionrunnerinfoprovider.py | 10 +- .../interfaces/iprojectfileclassifier.py | 6 +- .../interfaces/iprojectinfoprovider.py | 3 +- .../ipypackagelayoutinfoprovider.py | 23 + .../src/finecode_extension_api/service.py | 3 +- .../_services/run_action.py | 33 +- .../finecode_extension_runner/di/bootstrap.py | 49 ++- .../src/finecode_extension_runner/domain.py | 4 +- .../impls/command_runner.py | 2 +- .../impls/extension_runner_info_provider.py | 32 +- .../impls/project_file_classifier.py | 84 ++-- .../impls/project_info_provider.py | 4 +- .../finecode_extension_runner/lsp_server.py | 2 +- .../src/finecode_extension_runner/services.py | 8 +- src/finecode/base_config.toml | 19 +- src/finecode/cli.py | 4 +- src/finecode/cli_app/commands/__init__.py | 0 .../dump_config_cmd.py} | 14 +- .../prepare_envs_cmd.py} | 39 +- src/finecode/cli_app/commands/run_cmd.py | 193 +++++++++ src/finecode/cli_app/run.py | 402 ------------------ src/finecode/cli_app/utils.py | 119 ++++++ src/finecode/config/collect_actions.py | 1 - src/finecode/find_project.py | 10 +- .../lsp_server/endpoints/diagnostics.py | 31 +- .../lsp_server/endpoints/document_sync.py | 12 +- .../lsp_server/endpoints/formatting.py | 29 +- .../lsp_server/endpoints/inlay_hints.py | 5 +- src/finecode/lsp_server/lsp_server.py | 7 +- src/finecode/lsp_server/services.py | 3 +- src/finecode/project_analyzer.py | 174 +++----- src/finecode/runner/manager.py | 48 ++- src/finecode/runner/runner_client.py | 12 +- src/finecode/services.py | 168 -------- src/finecode/services/__init__.py | 3 + src/finecode/services/run_service/__init__.py | 12 + .../services/run_service/exceptions.py | 8 + .../run_service}/payload_preprocessor.py | 6 +- .../{ => services/run_service}/proxy_utils.py | 277 +++++++++++- src/finecode/services/shutdown.py | 20 + 53 files changed, 1208 insertions(+), 918 deletions(-) create mode 100644 extensions/fine_python_package_info/README.md create mode 100644 extensions/fine_python_package_info/fine_python_package_info/__init__.py create mode 100644 extensions/fine_python_package_info/fine_python_package_info/list_project_files_by_lang_python.py create mode 100644 extensions/fine_python_package_info/fine_python_package_info/py_package_layout_info_provider.py create mode 100644 extensions/fine_python_package_info/pyproject.toml create mode 100644 extensions/fine_python_package_info/setup.py create mode 100644 extensions/fine_python_package_info/tests/__init__.py create mode 100644 finecode_extension_api/src/finecode_extension_api/actions/list_project_files_by_lang.py create mode 100644 finecode_extension_api/src/finecode_extension_api/interfaces/ipypackagelayoutinfoprovider.py create mode 100644 src/finecode/cli_app/commands/__init__.py rename src/finecode/cli_app/{dump_config.py => commands/dump_config_cmd.py} (89%) rename src/finecode/cli_app/{prepare_envs.py => commands/prepare_envs_cmd.py} (89%) create mode 100644 src/finecode/cli_app/commands/run_cmd.py delete mode 100644 src/finecode/cli_app/run.py create mode 100644 src/finecode/cli_app/utils.py delete mode 100644 src/finecode/services.py create mode 100644 src/finecode/services/__init__.py create mode 100644 src/finecode/services/run_service/__init__.py create mode 100644 src/finecode/services/run_service/exceptions.py rename src/finecode/{ => services/run_service}/payload_preprocessor.py (92%) rename src/finecode/{ => services/run_service}/proxy_utils.py (54%) create mode 100644 src/finecode/services/shutdown.py diff --git a/extensions/fine_python_package_info/README.md b/extensions/fine_python_package_info/README.md new file mode 100644 index 0000000..e69de29 diff --git a/extensions/fine_python_package_info/fine_python_package_info/__init__.py b/extensions/fine_python_package_info/fine_python_package_info/__init__.py new file mode 100644 index 0000000..48da8ed --- /dev/null +++ b/extensions/fine_python_package_info/fine_python_package_info/__init__.py @@ -0,0 +1,7 @@ +from .list_project_files_by_lang_python import ListProjectFilesByLangPythonHandler +from .py_package_layout_info_provider import PyPackageLayoutInfoProvider + +__all__ = [ + "ListProjectFilesByLangPythonHandler", + "PyPackageLayoutInfoProvider" +] diff --git a/extensions/fine_python_package_info/fine_python_package_info/list_project_files_by_lang_python.py b/extensions/fine_python_package_info/fine_python_package_info/list_project_files_by_lang_python.py new file mode 100644 index 0000000..0116201 --- /dev/null +++ b/extensions/fine_python_package_info/fine_python_package_info/list_project_files_by_lang_python.py @@ -0,0 +1,48 @@ +from finecode_extension_api.interfaces import iprojectinfoprovider, ipypackagelayoutinfoprovider +import dataclasses +import pathlib + +from finecode_extension_api import code_action +from finecode_extension_api.actions import list_project_files_by_lang as list_project_files_by_lang_action + + +@dataclasses.dataclass +class ListProjectFilesByLangPythonHandlerConfig(code_action.ActionHandlerConfig): ... +# TODO: parameter for additional dirs + + +class ListProjectFilesByLangPythonHandler( + code_action.ActionHandler[ + list_project_files_by_lang_action.ListProjectFilesByLangAction, ListProjectFilesByLangPythonHandlerConfig + ] +): + def __init__(self, project_info_provider: iprojectinfoprovider.IProjectInfoProvider, py_package_layout_info_provider: ipypackagelayoutinfoprovider.IPyPackageLayoutInfoProvider) -> None: + self.project_info_provider = project_info_provider + self.py_package_layout_info_provider = py_package_layout_info_provider + + self.current_project_dir_path = self.project_info_provider.get_current_project_dir_path() + self.tests_dir_path = self.current_project_dir_path / 'tests' + self.scripts_dir_path = self.current_project_dir_path / 'scripts' + self.setup_py_path = self.current_project_dir_path / 'setup.py' + + async def run( + self, + payload: list_project_files_by_lang_action.ListProjectFilesByLangRunPayload, + run_context: list_project_files_by_lang_action.ListProjectFilesByLangRunContext, + ) -> list_project_files_by_lang_action.ListProjectFilesByLangRunResult: + py_files: list[pathlib.Path] = [] + project_package_src_root_dir_path = await self.py_package_layout_info_provider.get_package_src_root_dir_path(package_dir_path=self.current_project_dir_path) + py_files += list(project_package_src_root_dir_path.rglob('*.py')) + + if self.scripts_dir_path.exists(): + py_files += list(self.scripts_dir_path.rglob('*.py')) + + if self.tests_dir_path.exists(): + py_files += list(self.tests_dir_path.rglob('*.py')) + + if self.setup_py_path.exists(): + py_files.append(self.setup_py_path) + + return list_project_files_by_lang_action.ListProjectFilesByLangRunResult( + files_by_lang={"python": py_files} + ) diff --git a/extensions/fine_python_package_info/fine_python_package_info/py_package_layout_info_provider.py b/extensions/fine_python_package_info/fine_python_package_info/py_package_layout_info_provider.py new file mode 100644 index 0000000..c347db7 --- /dev/null +++ b/extensions/fine_python_package_info/fine_python_package_info/py_package_layout_info_provider.py @@ -0,0 +1,47 @@ +import enum +import pathlib + +import tomlkit + +from finecode_extension_api.interfaces import ifilemanager, ipypackagelayoutinfoprovider +from finecode_extension_api import service + + +class PyPackageLayoutInfoProvider(ipypackagelayoutinfoprovider.IPyPackageLayoutInfoProvider, service.Service): + def __init__(self, file_manager: ifilemanager.IFileManager) -> None: + self.file_manager = file_manager + # TODO: cache package name by file version? + + async def _get_package_name(self, package_dir_path: pathlib.Path) -> str: + package_def_file = package_dir_path / 'pyproject.toml' + if not package_def_file.exists(): + raise NotImplementedError("Only python packages with pyproject.toml config file are supported") + + package_def_file_content = await self.file_manager.get_content(file_path=package_def_file) + # TODO: handle errors + package_def_dict = tomlkit.loads(package_def_file_content) + package_raw_name = package_def_dict.get('project', {}).get('name', None) + if package_raw_name is None: + raise ValueError(f"package.name not found in {package_def_file}") + + return package_raw_name.replace('-', '_') + + async def get_package_layout(self, package_dir_path: pathlib.Path) -> ipypackagelayoutinfoprovider.PyPackageLayout: + if (package_dir_path / 'src').exists(): + return ipypackagelayoutinfoprovider.PyPackageLayout.SRC + else: + package_name = await self._get_package_name(package_dir_path=package_dir_path) + if (package_dir_path / package_name).exists(): + return ipypackagelayoutinfoprovider.PyPackageLayout.FLAT + else: + return ipypackagelayoutinfoprovider.PyPackageLayout.CUSTOM + + async def get_package_src_root_dir_path(self, package_dir_path: str) -> pathlib.Path: + package_layout = await self.get_package_layout(package_dir_path=package_dir_path) + package_name = await self._get_package_name(package_dir_path=package_dir_path) + if package_layout == ipypackagelayoutinfoprovider.PyPackageLayout.SRC: + return package_dir_path / 'src' / package_name + elif package_layout == ipypackagelayoutinfoprovider.PyPackageLayout.FLAT: + return package_dir_path / package_name + else: + raise NotImplementedError(f"Custom python package layout in {package_dir_path} is not supported") diff --git a/extensions/fine_python_package_info/pyproject.toml b/extensions/fine_python_package_info/pyproject.toml new file mode 100644 index 0000000..557f467 --- /dev/null +++ b/extensions/fine_python_package_info/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "fine_python_package_info" +version = "0.1.0" +description = "" +authors = [{ name = "Vladyslav Hnatiuk", email = "aders1234@gmail.com" }] +readme = "README.md" +requires-python = ">=3.11, < 3.14" +dependencies = ["finecode_extension_api==0.3.*", "tomlkit==0.11.*"] diff --git a/extensions/fine_python_package_info/setup.py b/extensions/fine_python_package_info/setup.py new file mode 100644 index 0000000..f573cbe --- /dev/null +++ b/extensions/fine_python_package_info/setup.py @@ -0,0 +1,67 @@ +import atexit +import shutil +import sys +import tempfile + +from setuptools import setup +from setuptools.command.build import build +from setuptools.command.build_ext import build_ext +from setuptools.command.build_py import build_py +from setuptools.command.egg_info import egg_info + +# Create a single temp directory for all build operations +_TEMP_BUILD_DIR = None + + +def get_temp_build_dir(pkg_name): + global _TEMP_BUILD_DIR + if _TEMP_BUILD_DIR is None: + _TEMP_BUILD_DIR = tempfile.mkdtemp(prefix=f"{pkg_name}_build_") + atexit.register(lambda: shutil.rmtree(_TEMP_BUILD_DIR, ignore_errors=True)) + return _TEMP_BUILD_DIR + + +class TempDirBuildMixin: + def initialize_options(self): + super().initialize_options() + temp_dir = get_temp_build_dir(self.distribution.get_name()) + self.build_base = temp_dir + + +class TempDirEggInfoMixin: + def initialize_options(self): + super().initialize_options() + temp_dir = get_temp_build_dir(self.distribution.get_name()) + self.egg_base = temp_dir + + +class CustomBuild(TempDirBuildMixin, build): + pass + + +class CustomBuildPy(TempDirBuildMixin, build_py): + pass + + +class CustomBuildExt(TempDirBuildMixin, build_ext): + pass + + +class CustomEggInfo(TempDirEggInfoMixin, egg_info): + def initialize_options(self): + # Don't use temp dir for editable installs + if "--editable" in sys.argv or "-e" in sys.argv: + egg_info.initialize_options(self) + else: + super().initialize_options() + + +setup( + name="fine_python_package_info", + cmdclass={ + "build": CustomBuild, + "build_py": CustomBuildPy, + "build_ext": CustomBuildExt, + "egg_info": CustomEggInfo, + }, +) \ No newline at end of file diff --git a/extensions/fine_python_package_info/tests/__init__.py b/extensions/fine_python_package_info/tests/__init__.py new file mode 100644 index 0000000..ddbc937 --- /dev/null +++ b/extensions/fine_python_package_info/tests/__init__.py @@ -0,0 +1 @@ +# Tests for fine_python_package_info extension \ No newline at end of file diff --git a/extensions/fine_python_pip/setup.py b/extensions/fine_python_pip/setup.py index a7659bf..4611898 100644 --- a/extensions/fine_python_pip/setup.py +++ b/extensions/fine_python_pip/setup.py @@ -9,6 +9,7 @@ from setuptools.command.build_py import build_py from setuptools.command.egg_info import egg_info + # Create a single temp directory for all build operations _TEMP_BUILD_DIR = None diff --git a/extensions/fine_python_ruff/fine_python_ruff/format_handler.py b/extensions/fine_python_ruff/fine_python_ruff/format_handler.py index 2bd0bab..f3ef0d7 100644 --- a/extensions/fine_python_ruff/fine_python_ruff/format_handler.py +++ b/extensions/fine_python_ruff/fine_python_ruff/format_handler.py @@ -37,7 +37,6 @@ def __init__( command_runner: icommandrunner.ICommandRunner, ) -> None: self.config = config - self.context = context self.logger = logger self.cache = cache self.command_runner = command_runner diff --git a/finecode_builtin_handlers/src/finecode_builtin_handlers/__init__.py b/finecode_builtin_handlers/src/finecode_builtin_handlers/__init__.py index 32bd0b4..d98bd89 100644 --- a/finecode_builtin_handlers/src/finecode_builtin_handlers/__init__.py +++ b/finecode_builtin_handlers/src/finecode_builtin_handlers/__init__.py @@ -16,4 +16,4 @@ "PrepareRunnersInstallRunnerAndPresetsHandler", "PrepareRunnersReadConfigsHandler", "DumpConfigSaveHandler", -] \ No newline at end of file +] diff --git a/finecode_extension_api/src/finecode_extension_api/actions/list_project_files_by_lang.py b/finecode_extension_api/src/finecode_extension_api/actions/list_project_files_by_lang.py new file mode 100644 index 0000000..dbcf60a --- /dev/null +++ b/finecode_extension_api/src/finecode_extension_api/actions/list_project_files_by_lang.py @@ -0,0 +1,53 @@ +import dataclasses +import pathlib +import sys +import typing + +if sys.version_info >= (3, 12): + from typing import override +else: + from typing_extensions import override + +from finecode_extension_api import code_action, textstyler + + +@dataclasses.dataclass +class ListProjectFilesByLangRunPayload(code_action.RunActionPayload): ... + + +class ListProjectFilesByLangRunContext(code_action.RunActionContext): + def __init__( + self, + run_id: int, + ) -> None: + super().__init__(run_id=run_id) + + +@dataclasses.dataclass +class ListProjectFilesByLangRunResult(code_action.RunActionResult): + files_by_lang: dict[str, list[pathlib.Path]] + + @override + def update(self, other: code_action.RunActionResult) -> None: + if not isinstance(other, ListProjectFilesByLangRunResult): + return + + for lang, files in other.files_by_lang.items(): + if lang not in self.files_by_lang: + self.files_by_lang[lang] = files + else: + self.files_by_lang[lang] += files + + def to_text(self) -> str | textstyler.StyledText: + formatted_result = textstyler.StyledText() + for language, files in self.files_by_lang.items(): + formatted_result.append_styled(text=language + "\n", bold=True) + for file_path in files: + formatted_result.append(file_path.as_posix() + "\n") + return formatted_result + + +class ListProjectFilesByLangAction(code_action.Action): + PAYLOAD_TYPE = ListProjectFilesByLangRunPayload + RUN_CONTEXT_TYPE = ListProjectFilesByLangRunContext + RESULT_TYPE = ListProjectFilesByLangRunResult diff --git a/finecode_extension_api/src/finecode_extension_api/actions/prepare_envs.py b/finecode_extension_api/src/finecode_extension_api/actions/prepare_envs.py index ff7b7e2..51ed001 100644 --- a/finecode_extension_api/src/finecode_extension_api/actions/prepare_envs.py +++ b/finecode_extension_api/src/finecode_extension_api/actions/prepare_envs.py @@ -46,9 +46,9 @@ def __init__( # for example additional dependencies should be installed by adding handler # which inserts them into project definition instead of modying `install_deps` # handler - self.project_def_by_venv_dir_path: dict[pathlib.Path, dict[str, typing.Any]] = ( - {} - ) + self.project_def_by_venv_dir_path: dict[ + pathlib.Path, dict[str, typing.Any] + ] = {} async def init(self, initial_payload: PrepareEnvsRunPayload) -> None: for env_info in initial_payload.envs: diff --git a/finecode_extension_api/src/finecode_extension_api/actions/prepare_runners.py b/finecode_extension_api/src/finecode_extension_api/actions/prepare_runners.py index ec1a9fc..a7c0329 100644 --- a/finecode_extension_api/src/finecode_extension_api/actions/prepare_runners.py +++ b/finecode_extension_api/src/finecode_extension_api/actions/prepare_runners.py @@ -46,9 +46,9 @@ def __init__( # for example additional dependencies should be installed by adding handler # which inserts them into project definition instead of modying `install_deps` # handler - self.project_def_by_venv_dir_path: dict[pathlib.Path, dict[str, typing.Any]] = ( - {} - ) + self.project_def_by_venv_dir_path: dict[ + pathlib.Path, dict[str, typing.Any] + ] = {} async def init(self, initial_payload: PrepareRunnersRunPayload) -> None: for env_info in initial_payload.envs: diff --git a/finecode_extension_api/src/finecode_extension_api/interfaces/iextensionrunnerinfoprovider.py b/finecode_extension_api/src/finecode_extension_api/interfaces/iextensionrunnerinfoprovider.py index 280c76f..61f45dd 100644 --- a/finecode_extension_api/src/finecode_extension_api/interfaces/iextensionrunnerinfoprovider.py +++ b/finecode_extension_api/src/finecode_extension_api/interfaces/iextensionrunnerinfoprovider.py @@ -5,8 +5,8 @@ class IExtensionRunnerInfoProvider(Protocol): def get_cache_dir_path(self) -> pathlib.Path: ... - def get_venv_dir_path_of_env(self, env_name: str) -> pathlib.Path: - ... - - def get_venv_site_packages(self, venv_dir_path: pathlib.Path) -> list[pathlib.Path]: - ... \ No newline at end of file + def get_venv_dir_path_of_env(self, env_name: str) -> pathlib.Path: ... + + def get_venv_site_packages( + self, venv_dir_path: pathlib.Path + ) -> list[pathlib.Path]: ... diff --git a/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectfileclassifier.py b/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectfileclassifier.py index dd15b7a..09d8bf4 100644 --- a/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectfileclassifier.py +++ b/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectfileclassifier.py @@ -10,8 +10,6 @@ class ProjectFileType(enum.Enum): class IProjectFileClassifier(typing.Protocol): - def get_project_file_type(self, file_path: pathlib.Path) -> ProjectFileType: - ... + def get_project_file_type(self, file_path: pathlib.Path) -> ProjectFileType: ... - def get_env_for_file_type(self, file_type: ProjectFileType) -> str: - ... + def get_env_for_file_type(self, file_type: ProjectFileType) -> str: ... diff --git a/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectinfoprovider.py b/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectinfoprovider.py index 02f1243..87d94ab 100644 --- a/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectinfoprovider.py +++ b/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectinfoprovider.py @@ -7,8 +7,7 @@ def get_current_project_dir_path(self) -> pathlib.Path: ... def get_current_project_def_path(self) -> pathlib.Path: ... - async def get_current_project_package_name(self) -> str: - ... + async def get_current_project_package_name(self) -> str: ... async def get_project_raw_config( self, project_def_path: pathlib.Path diff --git a/finecode_extension_api/src/finecode_extension_api/interfaces/ipypackagelayoutinfoprovider.py b/finecode_extension_api/src/finecode_extension_api/interfaces/ipypackagelayoutinfoprovider.py new file mode 100644 index 0000000..a337fdc --- /dev/null +++ b/finecode_extension_api/src/finecode_extension_api/interfaces/ipypackagelayoutinfoprovider.py @@ -0,0 +1,23 @@ +import enum +import pathlib +from typing import Protocol + + +class PyPackageLayout(enum.Enum): + SRC = enum.auto() + FLAT = enum.auto() + CUSTOM = enum.auto() + + +class IPyPackageLayoutInfoProvider(Protocol): + async def get_package_layout( + self, package_dir_path: pathlib.Path + ) -> PyPackageLayout: ... + + async def get_package_src_root_dir_path( + self, package_dir_path: str + ) -> pathlib.Path: + # returns path to root directory with package sources(where main __init__.py is). + # if you need path to directory which is added to sys.path during execution, take + # parent of this directory. + ... diff --git a/finecode_extension_api/src/finecode_extension_api/service.py b/finecode_extension_api/src/finecode_extension_api/service.py index 75592ce..1a66440 100644 --- a/finecode_extension_api/src/finecode_extension_api/service.py +++ b/finecode_extension_api/src/finecode_extension_api/service.py @@ -1,3 +1,2 @@ class Service: - async def init(self) -> None: - ... + async def init(self) -> None: ... diff --git a/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py b/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py index fc2d82b..7254891 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py +++ b/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py @@ -52,7 +52,7 @@ async def run_action( global last_run_id run_id = last_run_id last_run_id += 1 - logger.trace(f"Run action '{request.action_name}', run id: {run_id}") + logger.trace(f"Run action '{request.action_name}', run id: {run_id}, partial result token: {options.partial_result_token}") # TODO: check whether config is set: this will be solved by passing initial # configuration as payload of initialize if global_state.runner_context is None: @@ -195,14 +195,14 @@ async def run_action( # all subresults are ready logger.trace(f"R{run_id} | all subresults are ready, send them") await partial_result_sender.send_all_immediately() - - for subresult_task in subresults_tasks: - result = subresult_task.result() - if result is not None: - if action_result is None: - action_result = result - else: - action_result.update(result) + else: + for subresult_task in subresults_tasks: + result = subresult_task.result() + if result is not None: + if action_result is None: + action_result = result + else: + action_result.update(result) else: # action payload not iterable, just execute handlers on the whole payload if execute_handlers_concurrently: @@ -381,10 +381,10 @@ async def execute_action_handler( else: handler_cache = domain.ActionHandlerCache() action_cache.handler_cache_by_name[handler.name] = handler_cache - + start_time = time.time_ns() execution_result: code_action.RunActionResult | None = None - + handler_global_config = runner_context.project.action_handler_configs.get( handler.source, None ) @@ -538,11 +538,11 @@ async def run_subresult_coros_concurrently( errors_str = "" for exc in eg.exceptions: if isinstance(exc, code_action.ActionFailedException): - errors_str += exc.message + '.' + errors_str += exc.message + "." else: logger.error("Unhandled exception:") logger.exception(exc) - errors_str += str(exc) + '.' + errors_str += str(exc) + "." raise ActionFailedException( f"Concurrent running action handlers of '{action_name}' failed(Run {run_id}): {errors_str}" ) @@ -552,7 +552,12 @@ async def run_subresult_coros_concurrently( coro_result = coro_task.result() if coro_result is not None: if action_subresult is None: - action_subresult = coro_result + # copy the first result because all further subresults will be merged + # in it and result from action handler must stay immutable (e.g. it can + # reference to cache) + action_subresult_type = type(coro_result) + action_subresult_dict = dataclasses.asdict(coro_result) + action_subresult = action_subresult_type(**action_subresult_dict) else: action_subresult.update(coro_result) diff --git a/finecode_extension_runner/src/finecode_extension_runner/di/bootstrap.py b/finecode_extension_runner/src/finecode_extension_runner/di/bootstrap.py index 50bd24f..30e147e 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/di/bootstrap.py +++ b/finecode_extension_runner/src/finecode_extension_runner/di/bootstrap.py @@ -12,6 +12,11 @@ except ImportError: fine_python_mypy = None +try: + import fine_python_package_info +except ImportError: + fine_python_package_info = None + from finecode_extension_api.interfaces import ( iactionrunner, icache, @@ -20,7 +25,8 @@ ilogger, iprojectinfoprovider, iextensionrunnerinfoprovider, - iprojectfileclassifier + iprojectfileclassifier, + ipypackagelayoutinfoprovider, ) from finecode_extension_runner import global_state, schemas from finecode_extension_runner._services import run_action @@ -33,7 +39,7 @@ loguru_logger, project_info_provider, extension_runner_info_provider, - project_file_classifier + project_file_classifier, ) @@ -78,11 +84,20 @@ def bootstrap( project_def_path_getter=project_def_path_getter, project_raw_config_getter=project_raw_config_getter, ) - _state.factories[iextensionrunnerinfoprovider.IExtensionRunnerInfoProvider] = functools.partial( - extension_runner_info_provider_factory, - cache_dir_path_getter=cache_dir_path_getter + _state.factories[iextensionrunnerinfoprovider.IExtensionRunnerInfoProvider] = ( + functools.partial( + extension_runner_info_provider_factory, + cache_dir_path_getter=cache_dir_path_getter, + ) ) - _state.factories[iprojectfileclassifier.IProjectFileClassifier] = project_file_classifier_factory + _state.factories[iprojectfileclassifier.IProjectFileClassifier] = ( + project_file_classifier_factory + ) + + if fine_python_package_info is not None: + _state.factories[ipypackagelayoutinfoprovider.IPyPackageLayoutInfoProvider] = ( + py_package_layout_info_provider_factory + ) # TODO: parameters from config @@ -134,13 +149,27 @@ async def extension_runner_info_provider_factory( ): logger = await resolver.get_service_instance(ilogger.ILogger) return extension_runner_info_provider.ExtensionRunnerInfoProvider( - cache_dir_path_getter=cache_dir_path_getter, - logger=logger + cache_dir_path_getter=cache_dir_path_getter, logger=logger ) async def project_file_classifier_factory( container, ): - project_info_provider = await resolver.get_service_instance(iprojectinfoprovider.IProjectInfoProvider) - return project_file_classifier.ProjectFileClassifier(project_info_provider=project_info_provider) + project_info_provider = await resolver.get_service_instance( + iprojectinfoprovider.IProjectInfoProvider + ) + py_package_layout_info_provider = await resolver.get_service_instance( + ipypackagelayoutinfoprovider.IPyPackageLayoutInfoProvider + ) + return project_file_classifier.ProjectFileClassifier( + project_info_provider=project_info_provider, + py_package_layout_info_provider=py_package_layout_info_provider, + ) + + +async def py_package_layout_info_provider_factory(container): + file_manager = await resolver.get_service_instance(ifilemanager.IFileManager) + return fine_python_package_info.PyPackageLayoutInfoProvider( + file_manager=file_manager, + ) diff --git a/finecode_extension_runner/src/finecode_extension_runner/domain.py b/finecode_extension_runner/src/finecode_extension_runner/domain.py index c3a64a4..c96bb28 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/domain.py +++ b/finecode_extension_runner/src/finecode_extension_runner/domain.py @@ -82,7 +82,9 @@ class ActionHandlerExecInfoStatus(enum.Enum): @dataclasses.dataclass class ActionCache: exec_info: ActionExecInfo | None = None - handler_cache_by_name: dict[str, ActionHandlerCache] = dataclasses.field(default_factory=dict) + handler_cache_by_name: dict[str, ActionHandlerCache] = dataclasses.field( + default_factory=dict + ) @dataclasses.dataclass diff --git a/finecode_extension_runner/src/finecode_extension_runner/impls/command_runner.py b/finecode_extension_runner/src/finecode_extension_runner/impls/command_runner.py index 464c348..e9908b9 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/impls/command_runner.py +++ b/finecode_extension_runner/src/finecode_extension_runner/impls/command_runner.py @@ -121,7 +121,7 @@ def run_sync( cmd_parts = shlex.split(cmd) log_msg = f"Sync subprocess run: {cmd_parts}" if cwd is not None: - log_msg += f' {cwd}' + log_msg += f" {cwd}" self.logger.debug(log_msg) async_subprocess = subprocess.Popen( cmd_parts, diff --git a/finecode_extension_runner/src/finecode_extension_runner/impls/extension_runner_info_provider.py b/finecode_extension_runner/src/finecode_extension_runner/impls/extension_runner_info_provider.py index d379a62..281da0b 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/impls/extension_runner_info_provider.py +++ b/finecode_extension_runner/src/finecode_extension_runner/impls/extension_runner_info_provider.py @@ -4,15 +4,15 @@ from finecode_extension_api.interfaces import iextensionrunnerinfoprovider, ilogger -class ExtensionRunnerInfoProvider(iextensionrunnerinfoprovider.IExtensionRunnerInfoProvider): +class ExtensionRunnerInfoProvider( + iextensionrunnerinfoprovider.IExtensionRunnerInfoProvider +): def __init__( - self, - cache_dir_path_getter: Callable[[], pathlib.Path], - logger: ilogger.ILogger + self, cache_dir_path_getter: Callable[[], pathlib.Path], logger: ilogger.ILogger ) -> None: self.cache_dir_path_getter = cache_dir_path_getter self.logger = logger - + self._site_packages_cache: dict[pathlib.Path, list[pathlib.Path]] = {} def get_cache_dir_path(self) -> pathlib.Path: @@ -31,21 +31,27 @@ def get_venv_site_packages(self, venv_dir_path: pathlib.Path) -> list[pathlib.Pa # reloaded and cache will be automatically cleared if venv_dir_path in self._site_packages_cache: return self._site_packages_cache[venv_dir_path] - + site_packages: list[pathlib.Path] = [] - for lib_dir_name in ['lib', 'lib64']: + for lib_dir_name in ["lib", "lib64"]: lib_dir_path = venv_dir_path / lib_dir_name - + if not lib_dir_path.exists(): continue - + # assume there is only one python version in venv - lib_python_dir_path = next(dir_path for dir_path in lib_dir_path.iterdir() if dir_path.is_dir() and dir_path.name.startswith('python')) - site_packages_path = lib_python_dir_path / 'site-packages' + lib_python_dir_path = next( + dir_path + for dir_path in lib_dir_path.iterdir() + if dir_path.is_dir() and dir_path.name.startswith("python") + ) + site_packages_path = lib_python_dir_path / "site-packages" if site_packages_path.exists(): site_packages.append(site_packages_path) else: - self.logger.warning(f"site-packages directory expected in {lib_python_dir_path}, but wasn't exist. Venv seems to be invalid") - + self.logger.warning( + f"site-packages directory expected in {lib_python_dir_path}, but wasn't exist. Venv seems to be invalid" + ) + self._site_packages_cache[venv_dir_path] = site_packages return site_packages diff --git a/finecode_extension_runner/src/finecode_extension_runner/impls/project_file_classifier.py b/finecode_extension_runner/src/finecode_extension_runner/impls/project_file_classifier.py index 64097cc..b444f84 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/impls/project_file_classifier.py +++ b/finecode_extension_runner/src/finecode_extension_runner/impls/project_file_classifier.py @@ -1,63 +1,71 @@ import enum import pathlib -from finecode_extension_api.interfaces import iprojectfileclassifier, iprojectinfoprovider +from finecode_extension_api.interfaces import ( + iprojectfileclassifier, + iprojectinfoprovider, + ipypackagelayoutinfoprovider, +) from finecode_extension_api import service -class ProjectLayout(enum.Enum): - SRC = enum.auto() - FLAT = enum.auto() - CUSTOM = enum.auto() - - -class ProjectFileClassifier(iprojectfileclassifier.IProjectFileClassifier, service.Service): +# TODO: it should be package file classifier? +class ProjectFileClassifier( + iprojectfileclassifier.IProjectFileClassifier, service.Service +): # requirements: # - all project sources should be in a single directory # - if tests are outside of sources, they should be in a single directory + # + # Note: this service classifies files in root package of the project. It means if + # project contains multiple packages, they will be not considered - def __init__(self, project_info_provider: iprojectinfoprovider.IProjectInfoProvider) -> None: + def __init__( + self, + project_info_provider: iprojectinfoprovider.IProjectInfoProvider, + py_package_layout_info_provider: ipypackagelayoutinfoprovider.IPyPackageLayoutInfoProvider, + ) -> None: self.project_info_provider = project_info_provider + self.py_package_layout_info_provider = py_package_layout_info_provider # ProjectFileClassifier is instantiated as singletone, cache can be stored in # object - self._file_type_by_path: dict[pathlib.Path, iprojectfileclassifier.ProjectFileType] = {} + self._file_type_by_path: dict[ + pathlib.Path, iprojectfileclassifier.ProjectFileType + ] = {} - self.project_layout: ProjectLayout self.project_src_dir_path: pathlib.Path self.project_tests_dir_path: pathlib.Path async def init(self) -> None: project_dir_path = self.project_info_provider.get_current_project_dir_path() - project_package_name = await self.project_info_provider.get_current_project_package_name() - self.project_layout = self._get_project_layout(project_dir_path, project_package_name) - if self.project_layout == ProjectLayout.SRC: - self.project_src_dir_path = project_dir_path / 'src' - elif self.project_layout == ProjectLayout.FLAT: - self.project_src_dir_path = project_dir_path / project_package_name - else: - self.project_src_dir_path = None - - self.project_tests_dir_path: pathlib.Path = project_dir_path / 'tests' - def _get_project_layout(self, project_dir_path: pathlib.Path, project_package_name: str) -> ProjectLayout: - if (project_dir_path / 'src').exists(): - return ProjectLayout.SRC - elif (project_dir_path / project_package_name).exists(): - return ProjectLayout.FLAT - else: - return ProjectLayout.CUSTOM + self.project_src_dir_path = ( + await self.py_package_layout_info_provider.get_package_src_root_dir_path( + package_dir_path=project_dir_path + ) + ) + self.project_tests_dir_path: pathlib.Path = project_dir_path / "tests" - def get_project_file_type(self, file_path: pathlib.Path) -> iprojectfileclassifier.ProjectFileType: + def get_project_file_type( + self, file_path: pathlib.Path + ) -> iprojectfileclassifier.ProjectFileType: if self.project_src_dir_path is None: - raise NotImplementedError(f'{self.project_layout} project layout is not supported') + raise NotImplementedError( + f"{self.project_layout} project layout is not supported" + ) if file_path in self._file_type_by_path: # return cached value if exist return self._file_type_by_path[file_path] if file_path.is_relative_to(self.project_src_dir_path): - file_path_relative_to_project = file_path.relative_to(self.project_src_dir_path) - if '__tests__' in file_path_relative_to_project.parts or 'tests' in file_path_relative_to_project.parts: + file_path_relative_to_project = file_path.relative_to( + self.project_src_dir_path + ) + if ( + "__tests__" in file_path_relative_to_project.parts + or "tests" in file_path_relative_to_project.parts + ): file_type = iprojectfileclassifier.ProjectFileType.TEST else: file_type = iprojectfileclassifier.ProjectFileType.SOURCE @@ -73,15 +81,19 @@ def get_project_file_type(self, file_path: pathlib.Path) -> iprojectfileclassifi return file_type - def get_env_for_file_type(self, file_type: iprojectfileclassifier.ProjectFileType) -> str: + def get_env_for_file_type( + self, file_type: iprojectfileclassifier.ProjectFileType + ) -> str: match file_type: case iprojectfileclassifier.ProjectFileType.SOURCE: - return 'runtime' + return "runtime" case iprojectfileclassifier.ProjectFileType.TEST: # TODO: dynamic. In future test tool can be installed in any env, we # need a way to define it in config and get it here # TODO: there can be also e2e tests that don't use runtime and are in # e.g. dev_no_runtime env - return 'dev' + return "dev" case _: - raise NotImplementedError("") + raise NotImplementedError( + f"Project file type {file_type} is not supported by ProjectFileClassifier" + ) diff --git a/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py b/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py index 1a38dc2..9936972 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py +++ b/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py @@ -27,11 +27,11 @@ def get_current_project_def_path(self) -> pathlib.Path: async def get_current_project_package_name(self) -> str: project_raw_config = await self.get_current_project_raw_config() - raw_name = project_raw_config.get('project', {}).get('name', None) + raw_name = project_raw_config.get("project", {}).get("name", None) if raw_name is None: raise InvalidProjectConfig("project.name not found in project config") - return raw_name.replace('-', '_') + return raw_name.replace("-", "_") async def get_project_raw_config( self, project_def_path: pathlib.Path diff --git a/finecode_extension_runner/src/finecode_extension_runner/lsp_server.py b/finecode_extension_runner/src/finecode_extension_runner/lsp_server.py index 27b10b3..33b6edd 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/lsp_server.py +++ b/finecode_extension_runner/src/finecode_extension_runner/lsp_server.py @@ -70,9 +70,9 @@ def on_process_exit(): def send_partial_result( token: int | str, partial_result: code_action.RunActionResult ) -> None: - logger.debug(f"Send partial result for {token}") partial_result_dict = dataclasses.asdict(partial_result) partial_result_json = json.dumps(partial_result_dict) + logger.debug(f"Send partial result for {token}, length {len(partial_result_json)}") server.progress(types.ProgressParams(token=token, value=partial_result_json)) run_action_service.set_partial_result_sender(send_partial_result) diff --git a/finecode_extension_runner/src/finecode_extension_runner/services.py b/finecode_extension_runner/src/finecode_extension_runner/services.py index 4cbddad..415338a 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/services.py +++ b/finecode_extension_runner/src/finecode_extension_runner/services.py @@ -63,10 +63,12 @@ def project_def_path_getter() -> Path: def cache_dir_path_getter() -> Path: assert global_state.runner_context is not None project_dir_path = global_state.runner_context.project.dir_path - project_cache_dir = project_dir_path / ".venvs" / global_state.env_name / "cache" + project_cache_dir = ( + project_dir_path / ".venvs" / global_state.env_name / "cache" + ) if not project_cache_dir.exists(): project_cache_dir.mkdir() - + return project_cache_dir di_bootstrap.bootstrap( @@ -74,7 +76,7 @@ def cache_dir_path_getter() -> Path: save_document_func=document_saver, project_def_path_getter=project_def_path_getter, project_raw_config_getter=project_raw_config_getter, - cache_dir_path_getter=cache_dir_path_getter + cache_dir_path_getter=cache_dir_path_getter, ) return schemas.UpdateConfigResponse() diff --git a/src/finecode/base_config.toml b/src/finecode/base_config.toml index 06df1aa..0c59fd4 100644 --- a/src/finecode/base_config.toml +++ b/src/finecode/base_config.toml @@ -5,12 +5,14 @@ source = "finecode_extension_api.actions.prepare_envs.PrepareEnvsAction" name = "prepare_envs_dump_configs" source = "finecode_builtin_handlers.PrepareEnvsReadConfigsHandler" env = "dev_workspace" +dependencies = ["finecode_builtin_handlers==0.1.*"] [[tool.finecode.action.prepare_envs.handlers]] name = "prepare_envs_install_deps" source = "finecode_builtin_handlers.PrepareEnvsInstallDepsHandler" env = "dev_workspace" +dependencies = ["finecode_builtin_handlers==0.1.*"] # preparing dev workspaces doesn't need dumping config for two reasons: @@ -32,13 +34,14 @@ dependencies = ["fine_python_virtualenv==0.1.*"] name = "prepare_envs_read_configs" source = "finecode_builtin_handlers.PrepareEnvsReadConfigsHandler" env = "dev_workspace" +dependencies = ["finecode_builtin_handlers==0.1.*"] [[tool.finecode.action.prepare_dev_workspaces_envs.handlers]] name = "prepare_envs_install_deps" source = "finecode_builtin_handlers.PrepareEnvsInstallDepsHandler" env = "dev_workspace" - +dependencies = ["finecode_builtin_handlers==0.1.*"] [tool.finecode.action.prepare_runners] source = "finecode_extension_api.actions.prepare_runners.PrepareRunnersAction" @@ -53,12 +56,14 @@ dependencies = ["fine_python_virtualenv==0.1.*"] name = "prepare_runners_read_configs" source = "finecode_builtin_handlers.PrepareRunnersReadConfigsHandler" env = "dev_workspace" +dependencies = ["finecode_builtin_handlers==0.1.*"] [[tool.finecode.action.prepare_runners.handlers]] name = "prepare_runners_install_runner_and_presets" source = "finecode_builtin_handlers.PrepareRunnersInstallRunnerAndPresetsHandler" env = "dev_workspace" +dependencies = ["finecode_builtin_handlers==0.1.*"] [tool.finecode.action.dump_config] @@ -68,12 +73,14 @@ source = "finecode_extension_api.actions.dump_config.DumpConfigAction" name = "dump_config" source = "finecode_builtin_handlers.DumpConfigHandler" env = "dev_workspace" +dependencies = ["finecode_builtin_handlers==0.1.*"] [[tool.finecode.action.dump_config.handlers]] name = "dump_config_save" source = "finecode_builtin_handlers.DumpConfigSaveHandler" env = "dev_workspace" +dependencies = ["finecode_builtin_handlers==0.1.*"] [tool.finecode.action.install_deps_in_env] @@ -84,3 +91,13 @@ name = "install_deps_with_pip" source = "fine_python_pip.PipInstallDepsInEnvHandler" env = "dev_workspace" dependencies = ["fine_python_pip==0.1.*"] + + +[tool.finecode.action.list_project_files_by_lang] +source = "finecode_extension_api.actions.list_project_files_by_lang.ListProjectFilesByLangAction" + +[[tool.finecode.action.list_project_files_by_lang.handlers]] +name = "list_project_files_by_lang_python" +source = "fine_python_package_info.ListProjectFilesByLangPythonHandler" +env = "dev_no_runtime" +dependencies = ["fine_python_package_info==0.1.*"] diff --git a/src/finecode/cli.py b/src/finecode/cli.py index 6175b48..2385ffa 100644 --- a/src/finecode/cli.py +++ b/src/finecode/cli.py @@ -10,9 +10,7 @@ import finecode.main as workspace_manager from finecode import communication_utils, logger_utils, user_messages -from finecode.cli_app import dump_config as dump_config_cmd -from finecode.cli_app import prepare_envs as prepare_envs_cmd -from finecode.cli_app import run as run_cmd +from finecode.cli_app.commands import dump_config_cmd, prepare_envs_cmd, run_cmd @click.group() diff --git a/src/finecode/cli_app/commands/__init__.py b/src/finecode/cli_app/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/finecode/cli_app/dump_config.py b/src/finecode/cli_app/commands/dump_config_cmd.py similarity index 89% rename from src/finecode/cli_app/dump_config.py rename to src/finecode/cli_app/commands/dump_config_cmd.py index 0e25470..d2c6662 100644 --- a/src/finecode/cli_app/dump_config.py +++ b/src/finecode/cli_app/commands/dump_config_cmd.py @@ -1,10 +1,10 @@ -import os import pathlib from loguru import logger -from finecode import context, proxy_utils, services -from finecode.config import collect_actions, config_models, read_configs +from finecode import context, services +from finecode.services import run_service +from finecode.config import config_models, read_configs from finecode.runner import manager as runner_manager @@ -62,17 +62,17 @@ async def dump_config(workdir_path: pathlib.Path, project_name: str): ) try: - await proxy_utils.start_required_environments( + await run_service.start_required_environments( actions_by_projects, ws_context ) - except proxy_utils.StartingEnvironmentsFailed as exception: + except run_service.StartingEnvironmentsFailed as exception: raise DumpFailed( f"Failed to start environments for running 'dump_config': {exception.message}" ) project_raw_config = ws_context.ws_projects_raw_configs[project_dir_path] - await services.run_action( + await run_service.run_action( action_name="dump_config", params={ "source_file_path": project_def.def_path, @@ -81,7 +81,7 @@ async def dump_config(workdir_path: pathlib.Path, project_name: str): }, project_def=project_def, ws_context=ws_context, - result_format=services.RunResultFormat.STRING, + result_format=run_service.RunResultFormat.STRING, preprocess_payload=False, ) logger.info(f"Dumped config into {dump_file_path}") diff --git a/src/finecode/cli_app/prepare_envs.py b/src/finecode/cli_app/commands/prepare_envs_cmd.py similarity index 89% rename from src/finecode/cli_app/prepare_envs.py rename to src/finecode/cli_app/commands/prepare_envs_cmd.py index 2ffd73d..c581a91 100644 --- a/src/finecode/cli_app/prepare_envs.py +++ b/src/finecode/cli_app/commands/prepare_envs_cmd.py @@ -3,8 +3,9 @@ from loguru import logger -from finecode import context, domain, proxy_utils, services -from finecode.cli_app import run as run_cli +from finecode import context, domain, services +from finecode.services import run_service +from finecode.cli_app import utils from finecode.config import collect_actions, config_models, read_configs from finecode.runner import manager as runner_manager @@ -93,21 +94,22 @@ async def prepare_envs(workdir_path: pathlib.Path, recreate: bool) -> None: action_payload: dict[str, str | bool] = {"recreate": recreate} try: - await proxy_utils.start_required_environments( + await run_service.start_required_environments( actions_by_projects, ws_context ) - except proxy_utils.StartingEnvironmentsFailed as exception: + except run_service.StartingEnvironmentsFailed as exception: raise PrepareEnvsFailed( f"Failed to start environments for running 'prepare_runners': {exception.message}" ) try: - (result_output, result_return_code) = ( - await run_cli.run_actions_in_all_projects( - actions_by_projects, action_payload, ws_context, concurrently=True - ) + ( + result_output, + result_return_code, + ) = await utils.run_actions_in_projects_and_concat_results( + actions_by_projects, action_payload, ws_context, concurrently=True ) - except run_cli.RunFailed as error: + except run_service.ActionRunFailed as error: logger.error(error.message) result_output = error.message result_return_code = 1 @@ -122,12 +124,13 @@ async def prepare_envs(workdir_path: pathlib.Path, recreate: bool) -> None: action_payload: dict[str, str | bool] = {"recreate": recreate} try: - (result_output, result_return_code) = ( - await run_cli.run_actions_in_all_projects( - actions_by_projects, action_payload, ws_context, concurrently=True - ) + ( + result_output, + result_return_code, + ) = await utils.run_actions_in_projects_and_concat_results( + actions_by_projects, action_payload, ws_context, concurrently=True ) - except run_cli.RunFailed as error: + except run_service.ActionRunFailed as error: logger.error(error.message) result_output = error.message result_return_code = 1 @@ -189,7 +192,7 @@ async def check_or_recreate_all_dev_workspace_envs( current_project_dir_path = ws_context.ws_dirs_paths[0] current_project = ws_context.ws_projects[current_project_dir_path] try: - runner = await runner_manager.start_runner( + await runner_manager.start_runner( project_def=current_project, env_name="dev_workspace", ws_context=ws_context ) except runner_manager.RunnerFailedToStart as exception: @@ -241,17 +244,17 @@ async def check_or_recreate_all_dev_workspace_envs( envs += invalid_envs try: - action_result = await services.run_action( + action_result = await run_service.run_action( action_name="prepare_dev_workspaces_envs", params={ "envs": envs, }, project_def=current_project, ws_context=ws_context, - result_format=services.RunResultFormat.STRING, + result_format=run_service.RunResultFormat.STRING, preprocess_payload=False, ) - except services.ActionRunFailed as exception: + except run_service.ActionRunFailed as exception: raise PrepareEnvsFailed( f"'prepare_dev_workspaces_env' failed in {current_project.name}: {exception.message}" ) diff --git a/src/finecode/cli_app/commands/run_cmd.py b/src/finecode/cli_app/commands/run_cmd.py new file mode 100644 index 0000000..a0d725a --- /dev/null +++ b/src/finecode/cli_app/commands/run_cmd.py @@ -0,0 +1,193 @@ +import asyncio +import pathlib +from typing import NamedTuple + +import click +import ordered_set +from loguru import logger + +from finecode import context, domain, services +from finecode.services import run_service +from finecode.config import collect_actions, config_models, read_configs +from finecode.runner import manager as runner_manager + +from finecode.cli_app import utils + + +class RunFailed(Exception): + def __init__(self, message: str) -> None: + self.message = message + + +async def run_actions( + workdir_path: pathlib.Path, + projects_names: list[str] | None, + actions: list[str], + action_payload: dict[str, str], + concurrently: bool, +) -> tuple[str, int]: + ws_context = context.WorkspaceContext([workdir_path]) + await read_configs.read_projects_in_dir( + dir_path=workdir_path, ws_context=ws_context + ) + + if projects_names is not None: + # projects are provided. Filter out other projects if there are more, they would + # not be used (run can be started in a workspace with also other projects) + ws_context.ws_projects = { + project_dir_path: project + for project_dir_path, project in ws_context.ws_projects.items() + if project.name in projects_names + } + + # make sure all projects use finecode + config_problem_found = False + for project in ws_context.ws_projects.values(): + if project.status != domain.ProjectStatus.CONFIG_VALID: + if project.status == domain.ProjectStatus.NO_FINECODE: + logger.error( + f"You asked to run action in project '{project.name}', but finecode is not used in it(=there is no 'dev_workspace' environment with 'finecode' package in it)" + ) + config_problem_found = True + elif project.status == domain.ProjectStatus.CONFIG_INVALID: + logger.error( + f"You asked to run action in project '{project.name}', but its configuration is invalid(see logs above for more details)" + ) + config_problem_found = True + else: + logger.error( + f"You asked to run action in project '{project.name}', but it has unexpected status: {project.status}" + ) + config_problem_found = True + + if config_problem_found: + raise RunFailed( + "There is a problem with configuration. See previous messages for more details" + ) + else: + # filter out packages that don't use finecode + ws_context.ws_projects = { + project_dir_path: project + for project_dir_path, project in ws_context.ws_projects.items() + if project.status != domain.ProjectStatus.NO_FINECODE + } + + # check that configuration of packages that use finecode is valid + config_problem_found = False + for project in ws_context.ws_projects.values(): + if project.status == domain.ProjectStatus.CONFIG_VALID: + continue + elif project.status == domain.ProjectStatus.CONFIG_INVALID: + logger.error( + f"Project '{project.name}' has invalid config, see messages above for more details" + ) + config_problem_found = True + else: + logger.error( + f"Project '{project.name}' has unexpected status: {project.status}" + ) + config_problem_found = True + + if config_problem_found: + raise RunFailed( + "There is a problem with configuration. See previous messages for more details" + ) + + projects: list[domain.Project] = [] + if projects_names is not None: + projects = get_projects_by_names(projects_names, ws_context, workdir_path) + else: + projects = list(ws_context.ws_projects.values()) + + # first read configs without presets to be able to start runners with presets + for project in projects: + try: + await read_configs.read_project_config( + project=project, ws_context=ws_context, resolve_presets=False + ) + collect_actions.collect_actions( + project_path=project.dir_path, ws_context=ws_context + ) + except config_models.ConfigurationError as exception: + raise RunFailed( + f"Reading project config and collecting actions in {project.dir_path} failed: {exception.message}" + ) + + try: + # 1. Start runners with presets to be able to resolve presets. Presets are + # required to be able to collect all actions, actions handlers and configs. + try: + await runner_manager.start_runners_with_presets(projects, ws_context) + except runner_manager.RunnerFailedToStart as exception: + raise RunFailed( + f"One or more projects are misconfigured, runners for them didn't" + f" start: {exception.message}. Check logs for details." + ) + except Exception as exception: + logger.error("Unexpected exception:") + logger.exception(exception) + + actions_by_projects: dict[pathlib.Path, list[str]] = {} + if projects_names is not None: + # check that all projects have all actions to detect problem and provide + # feedback as early as possible + actions_set: ordered_set.OrderedSet[str] = ordered_set.OrderedSet(actions) + for project in projects: + project_actions_set: ordered_set.OrderedSet[str] = ( + ordered_set.OrderedSet([action.name for action in project.actions]) + ) + missing_actions = actions_set - project_actions_set + if len(missing_actions) > 0: + raise RunFailed( + f"Actions {', '.join(missing_actions)} not found in project '{project.name}'" + ) + actions_by_projects[project.dir_path] = actions + else: + # no explicit project, run in `workdir`, it's expected to be a ws dir and + # actions will be run in all projects inside + actions_by_projects = run_service.find_projects_with_actions( + ws_context, actions + ) + + try: + await run_service.start_required_environments( + actions_by_projects, ws_context, update_config_in_running_runners=True + ) + except run_service.StartingEnvironmentsFailed as exception: + raise RunFailed( + f"Failed to start environments for running actions: {exception.message}" + ) + + try: + return await utils.run_actions_in_projects_and_concat_results( + actions_by_projects, action_payload, ws_context, concurrently + ) + except run_service.ActionRunFailed as exception: + raise RunFailed(f"Failed to run actions: {exception.message}") + finally: + services.on_shutdown(ws_context) + + +def get_projects_by_names( + projects_names: list[str], + ws_context: context.WorkspaceContext, + workdir_path: pathlib.Path, +) -> list[domain.Project]: + projects: list[domain.Project] = [] + for project_name in projects_names: + try: + project = next( + project + for project in ws_context.ws_projects.values() + if project.name == project_name + ) + except StopIteration: + raise RunFailed( + f"Project '{projects_names}' not found in working directory '{workdir_path}'" + ) + + projects.append(project) + return projects + + +__all__ = ["run_actions"] diff --git a/src/finecode/cli_app/run.py b/src/finecode/cli_app/run.py deleted file mode 100644 index cbd6a0d..0000000 --- a/src/finecode/cli_app/run.py +++ /dev/null @@ -1,402 +0,0 @@ -import asyncio -import pathlib -from typing import NamedTuple - -import click -import ordered_set -from loguru import logger - -from finecode import context, domain, proxy_utils, services -from finecode.config import collect_actions, config_models, read_configs -from finecode.runner import manager as runner_manager -from finecode.runner import runner_info - - -class RunFailed(Exception): - def __init__(self, message: str) -> None: - self.message = message - - -async def run_actions( - workdir_path: pathlib.Path, - projects_names: list[str] | None, - actions: list[str], - action_payload: dict[str, str], - concurrently: bool, -) -> tuple[str, int]: - ws_context = context.WorkspaceContext([workdir_path]) - await read_configs.read_projects_in_dir( - dir_path=workdir_path, ws_context=ws_context - ) - - if projects_names is not None: - # projects are provided. Filter out other projects if there are more, they would - # not be used (run can be started in a workspace with also other projects) - ws_context.ws_projects = { - project_dir_path: project - for project_dir_path, project in ws_context.ws_projects.items() - if project.name in projects_names - } - - # make sure all projects use finecode - config_problem_found = False - for project in ws_context.ws_projects.values(): - if project.status != domain.ProjectStatus.CONFIG_VALID: - if project.status == domain.ProjectStatus.NO_FINECODE: - logger.error( - f"You asked to run action in project '{project.name}', but finecode is not used in it(=there is no 'dev_workspace' environment with 'finecode' package in it)" - ) - config_problem_found = True - elif project.status == domain.ProjectStatus.CONFIG_INVALID: - logger.error( - f"You asked to run action in project '{project.name}', but its configuration is invalid(see logs above for more details)" - ) - config_problem_found = True - else: - logger.error( - f"You asked to run action in project '{project.name}', but it has unexpected status: {project.status}" - ) - config_problem_found = True - - if config_problem_found: - raise RunFailed( - "There is a problem with configuration. See previous messages for more details" - ) - else: - # filter out packages that don't use finecode - ws_context.ws_projects = { - project_dir_path: project - for project_dir_path, project in ws_context.ws_projects.items() - if project.status != domain.ProjectStatus.NO_FINECODE - } - - # check that configuration of packages that use finecode is valid - config_problem_found = False - for project in ws_context.ws_projects.values(): - if project.status == domain.ProjectStatus.CONFIG_VALID: - continue - elif project.status == domain.ProjectStatus.CONFIG_INVALID: - logger.error( - f"Project '{project.name}' has invalid config, see messages above for more details" - ) - config_problem_found = True - else: - logger.error( - f"Project '{project.name}' has unexpected status: {project.status}" - ) - config_problem_found = True - - if config_problem_found: - raise RunFailed( - "There is a problem with configuration. See previous messages for more details" - ) - - projects: list[domain.Project] = [] - if projects_names is not None: - projects = get_projects_by_names(projects_names, ws_context, workdir_path) - else: - projects = list(ws_context.ws_projects.values()) - - # first read configs without presets to be able to start runners with presets - for project in projects: - try: - await read_configs.read_project_config( - project=project, ws_context=ws_context, resolve_presets=False - ) - collect_actions.collect_actions( - project_path=project.dir_path, ws_context=ws_context - ) - except config_models.ConfigurationError as exception: - raise RunFailed( - f"Reading project config and collecting actions in {project.dir_path} failed: {exception.message}" - ) - - try: - # 1. Start runners with presets to be able to resolve presets. Presets are - # required to be able to collect all actions, actions handlers and configs. - try: - await runner_manager.start_runners_with_presets(projects, ws_context) - except runner_manager.RunnerFailedToStart as exception: - raise RunFailed( - f"One or more projects are misconfigured, runners for them didn't" - f" start: {exception.message}. Check logs for details." - ) - except Exception as exception: - logger.error("Unexpected exception:") - logger.exception(exception) - - actions_by_projects: dict[pathlib.Path, list[str]] = {} - if projects_names is not None: - # check that all projects have all actions to detect problem and provide - # feedback as early as possible - actions_set: ordered_set.OrderedSet[str] = ordered_set.OrderedSet(actions) - for project in projects: - project_actions_set: ordered_set.OrderedSet[str] = ( - ordered_set.OrderedSet([action.name for action in project.actions]) - ) - missing_actions = actions_set - project_actions_set - if len(missing_actions) > 0: - raise RunFailed( - f"Actions {', '.join(missing_actions)} not found in project '{project.name}'" - ) - actions_by_projects[project.dir_path] = actions - else: - # no explicit project, run in `workdir`, it's expected to be a ws dir and - # actions will be run in all projects inside - actions_by_projects = find_projects_with_actions(ws_context, actions) - - try: - await proxy_utils.start_required_environments( - actions_by_projects, ws_context, update_config_in_running_runners=True - ) - except proxy_utils.StartingEnvironmentsFailed as exception: - raise RunFailed( - f"Failed to start environments for running actions: {exception.message}" - ) - - return await run_actions_in_all_projects( - actions_by_projects, action_payload, ws_context, concurrently - ) - finally: - services.on_shutdown(ws_context) - - -def get_projects_by_names( - projects_names: list[str], - ws_context: context.WorkspaceContext, - workdir_path: pathlib.Path, -) -> list[domain.Project]: - projects: list[domain.Project] = [] - for project_name in projects_names: - try: - project = next( - project - for project in ws_context.ws_projects.values() - if project.name == project_name - ) - except StopIteration: - raise RunFailed( - f"Project '{projects_names}' not found in working directory '{workdir_path}'" - ) - - projects.append(project) - return projects - - -def find_projects_with_actions( - ws_context: context.WorkspaceContext, actions: list[str] -) -> dict[pathlib.Path, list[str]]: - actions_by_project: dict[pathlib.Path, list[str]] = {} - actions_set = ordered_set.OrderedSet(actions) - - for project in ws_context.ws_projects.values(): - project_actions_names = [action.name for action in project.actions] - # find which of requested actions are available in the project - action_to_run_in_project = actions_set & ordered_set.OrderedSet( - project_actions_names - ) - relevant_actions_in_project = list(action_to_run_in_project) - if len(relevant_actions_in_project) > 0: - actions_by_project[project.dir_path] = relevant_actions_in_project - - return actions_by_project - - -def run_result_to_str( - run_result: str | dict[str, list[str | dict[str, str | bool]]], action_name: str -) -> str: - run_result_str = "" - if isinstance(run_result, str): - run_result_str = run_result - elif isinstance(run_result, dict): - # styled text - text_parts = run_result.get("parts", []) - if not isinstance(text_parts, list): - raise RunFailed( - f"Running of action {action_name} failed: got unexpected result, 'parts' value expected to be a list." - ) - - for text_part in text_parts: - if isinstance(text_part, str): - run_result_str += text_part - elif isinstance(text_part, dict): - try: - text = text_part["text"] - except KeyError: - raise RunFailed( - f"Running of action {action_name} failed: got unexpected result, 'text' value is required in object with styled text params." - ) - - style_params: dict[str, str | bool] = {} - if "foreground" in text_part and isinstance( - text_part["foreground"], str - ): - style_params["fg"] = text_part["foreground"] - - if "background" in text_part and isinstance( - text_part["background"], str - ): - style_params["bg"] = text_part["background"] - - if "bold" in text_part and isinstance(text_part["bold"], bool): - style_params["bold"] = text_part["bold"] - - if "underline" in text_part and isinstance( - text_part["underline"], bool - ): - style_params["underline"] = text_part["underline"] - - if "overline" in text_part and isinstance(text_part["overline"], bool): - style_params["overline"] = text_part["overline"] - - if "italic" in text_part and isinstance(text_part["italic"], bool): - style_params["italic"] = text_part["italic"] - - if "blink" in text_part and isinstance(text_part["blink"], bool): - style_params["blink"] = text_part["blink"] - - if "strikethrough" in text_part and isinstance( - text_part["strikethrough"], bool - ): - style_params["strikethrough"] = text_part["strikethrough"] - - if "reset" in text_part and isinstance(text_part["reset"], bool): - style_params["reset"] = text_part["reset"] - - run_result_str += click.style(text, **style_params) - else: - raise RunFailed( - f"Running of action {action_name} failed: got unexpected result, 'parts' list can contain only strings or objects with styled text." - ) - - return run_result_str - - -class ActionRunResult(NamedTuple): - output: str - return_code: int - - -async def run_actions_in_running_project( - actions: list[str], - action_payload: dict[str, str], - project: domain.Project, - ws_context: context.WorkspaceContext, - concurrently: bool, -) -> dict[str, ActionRunResult]: - result_by_action: dict[str, ActionRunResult] = {} - - if concurrently: - run_tasks: list[asyncio.Task] = [] - try: - async with asyncio.TaskGroup() as tg: - for action_name in actions: - run_task = tg.create_task( - services.run_action( - action_name=action_name, - params=action_payload, - project_def=project, - ws_context=ws_context, - result_format=services.RunResultFormat.STRING, - ) - ) - run_tasks.append(run_task) - except ExceptionGroup as eg: - for exception in eg.exceptions: - if isinstance(exception, services.ActionRunFailed): - logger.error(f"{exception.message} in {project.name}") - else: - logger.error("Unexpected exception:") - logger.exception(exception) - raise RunFailed(f"Running of actions {actions} failed") - - for idx, run_task in enumerate(run_tasks): - run_result = run_task.result() - action_name = actions[idx] - run_result_str = run_result_to_str(run_result.result, action_name) - result_by_action[action_name] = ActionRunResult( - output=run_result_str, return_code=run_result.return_code - ) - else: - for action_name in actions: - try: - run_result = await services.run_action( - action_name=action_name, - params=action_payload, - project_def=project, - ws_context=ws_context, - result_format=services.RunResultFormat.STRING, - ) - except services.ActionRunFailed as exception: - raise RunFailed( - f"Running of action {action_name} failed: {exception.message}" - ) - except Exception as error: - logger.error("Unexpected exception") - logger.exception(error) - raise RunFailed( - f"Running of action {action_name} failed with unexpected exception" - ) - - run_result_str = run_result_to_str(run_result.result, action_name) - result_by_action[action_name] = ActionRunResult( - output=run_result_str, return_code=run_result.return_code - ) - - return result_by_action - - -async def run_actions_in_all_projects( - actions_by_project: dict[pathlib.Path, list[str]], - action_payload: dict[str, str], - ws_context: context.WorkspaceContext, - concurrently: bool, -) -> tuple[str, int]: - project_handler_tasks: list[asyncio.Task] = [] - try: - async with asyncio.TaskGroup() as tg: - for project_dir_path, actions_to_run in actions_by_project.items(): - project = ws_context.ws_projects[project_dir_path] - project_task = tg.create_task( - run_actions_in_running_project( - actions=actions_to_run, - action_payload=action_payload, - project=project, - ws_context=ws_context, - concurrently=concurrently, - ) - ) - project_handler_tasks.append(project_task) - except ExceptionGroup as eg: - for exception in eg.exceptions: - # TODO: merge all in one? - raise exception - - result_output: str = "" - result_return_code: int = 0 - - run_in_many_projects = len(actions_by_project) > 1 - projects_paths = list(actions_by_project.keys()) - for idx, project_task in enumerate(project_handler_tasks): - project_dir_path = projects_paths[idx] - result_by_action = project_task.result() - run_many_actions = len(result_by_action) > 1 - - if idx > 0: - result_output += "\n" - - if run_in_many_projects: - result_output += ( - f"{click.style(str(project_dir_path), bold=True, underline=True)}\n" - ) - - for action_name, action_result in result_by_action.items(): - if run_many_actions: - result_output += f"{click.style(action_name, bold=True)}:" - result_output += action_result.output - result_return_code |= action_result.return_code - - return (result_output, result_return_code) - - -__all__ = ["run_actions"] diff --git a/src/finecode/cli_app/utils.py b/src/finecode/cli_app/utils.py new file mode 100644 index 0000000..bd3dcaa --- /dev/null +++ b/src/finecode/cli_app/utils.py @@ -0,0 +1,119 @@ +import pathlib + +import click + +from finecode import context +from finecode.services import run_service + + +def run_result_to_str( + run_result: str | dict[str, list[str | dict[str, str | bool]]], action_name: str +) -> str: + run_result_str = "" + if isinstance(run_result, str): + run_result_str = run_result + elif isinstance(run_result, dict): + # styled text + text_parts = run_result.get("parts", []) + if not isinstance(text_parts, list): + raise run_service.ActionRunFailed( + f"Running of action {action_name} failed: got unexpected result, 'parts' value expected to be a list." + ) + + for text_part in text_parts: + if isinstance(text_part, str): + run_result_str += text_part + elif isinstance(text_part, dict): + try: + text = text_part["text"] + except KeyError: + raise run_service.ActionRunFailed( + f"Running of action {action_name} failed: got unexpected result, 'text' value is required in object with styled text params." + ) + + style_params: dict[str, str | bool] = {} + if "foreground" in text_part and isinstance( + text_part["foreground"], str + ): + style_params["fg"] = text_part["foreground"] + + if "background" in text_part and isinstance( + text_part["background"], str + ): + style_params["bg"] = text_part["background"] + + if "bold" in text_part and isinstance(text_part["bold"], bool): + style_params["bold"] = text_part["bold"] + + if "underline" in text_part and isinstance( + text_part["underline"], bool + ): + style_params["underline"] = text_part["underline"] + + if "overline" in text_part and isinstance(text_part["overline"], bool): + style_params["overline"] = text_part["overline"] + + if "italic" in text_part and isinstance(text_part["italic"], bool): + style_params["italic"] = text_part["italic"] + + if "blink" in text_part and isinstance(text_part["blink"], bool): + style_params["blink"] = text_part["blink"] + + if "strikethrough" in text_part and isinstance( + text_part["strikethrough"], bool + ): + style_params["strikethrough"] = text_part["strikethrough"] + + if "reset" in text_part and isinstance(text_part["reset"], bool): + style_params["reset"] = text_part["reset"] + + run_result_str += click.style(text, **style_params) + else: + raise run_service.ActionRunFailed( + f"Running of action {action_name} failed: got unexpected result, 'parts' list can contain only strings or objects with styled text." + ) + + return run_result_str + + +async def run_actions_in_projects_and_concat_results( + actions_by_project: dict[pathlib.Path, list[str]], + action_payload: dict[str, str], + ws_context: context.WorkspaceContext, + concurrently: bool, +) -> tuple[str, int]: + result_by_project = await run_service.run_actions_in_projects( + actions_by_project=actions_by_project, + action_payload=action_payload, + ws_context=ws_context, + concurrently=concurrently, + result_format=run_service.RunResultFormat.STRING, + ) + + result_output: str = "" + result_return_code: int = 0 + + run_in_many_projects = len(result_by_project) > 1 + is_first_project = True + for project_dir_path, result_by_action in result_by_project.items(): + run_many_actions = len(result_by_action) > 1 + + if not is_first_project: + result_output += "\n" + + if run_in_many_projects: + result_output += ( + f"{click.style(str(project_dir_path), bold=True, underline=True)}\n" + ) + + for action_name, action_result in result_by_action.items(): + if run_many_actions: + result_output += f"{click.style(action_name, bold=True)}:" + action_result_str = run_result_to_str(action_result.result, action_name) + result_output += action_result_str + result_return_code |= action_result.return_code + + if is_first_project: + is_first_project = False + + return (result_output, result_return_code) diff --git a/src/finecode/config/collect_actions.py b/src/finecode/config/collect_actions.py index 06bf796..669c689 100644 --- a/src/finecode/config/collect_actions.py +++ b/src/finecode/config/collect_actions.py @@ -60,7 +60,6 @@ def _collect_actions_in_config( for action_name, action_def_raw in ( config["tool"]["finecode"].get("action", {}).items() ): - try: action_def = config_models.ActionDefinition(**action_def_raw) except config_models.ValidationError as exception: diff --git a/src/finecode/find_project.py b/src/finecode/find_project.py index 1f074db..f0ba83d 100644 --- a/src/finecode/find_project.py +++ b/src/finecode/find_project.py @@ -73,7 +73,9 @@ async def find_project_with_action_for_file( else: if project.status == domain.ProjectStatus.CONFIG_VALID: try: - await runner_manager.get_or_start_runners_with_presets(project_dir_path=project_dir_path, ws_context=ws_context) + await runner_manager.get_or_start_runners_with_presets( + project_dir_path=project_dir_path, ws_context=ws_context + ) except runner_manager.RunnerFailedToStart as exception: raise ValueError( f"Action is related to project {project_dir_path} but runner " @@ -93,9 +95,9 @@ async def find_project_with_action_for_file( except StopIteration: continue - ws_context.project_path_by_dir_and_action[dir_path_str][ - action_name - ] = project_dir_path + ws_context.project_path_by_dir_and_action[dir_path_str][action_name] = ( + project_dir_path + ) return project_dir_path raise FileHasNotActionException( diff --git a/src/finecode/lsp_server/endpoints/diagnostics.py b/src/finecode/lsp_server/endpoints/diagnostics.py index f83de92..cf01a47 100644 --- a/src/finecode/lsp_server/endpoints/diagnostics.py +++ b/src/finecode/lsp_server/endpoints/diagnostics.py @@ -14,10 +14,10 @@ context, domain, project_analyzer, - proxy_utils, pygls_types_utils, services, ) +from finecode.services import run_service from finecode.lsp_server import global_state from finecode_extension_api.actions import lint as lint_action @@ -61,7 +61,7 @@ async def document_diagnostic_with_full_result( ) -> types.DocumentDiagnosticReport | None: logger.trace(f"Document diagnostic with full result: {file_path}") try: - response = await proxy_utils.find_action_project_and_run( + response = await run_service.find_action_project_and_run( file_path=file_path, action_name="lint", # TODO: use payload class @@ -70,7 +70,7 @@ async def document_diagnostic_with_full_result( }, ws_context=global_state.ws_context, ) - except proxy_utils.ActionRunFailed as error: + except run_service.ActionRunFailed as error: # don't throw error because vscode after a few sequential errors will stop # requesting diagnostics until restart. Show user message instead logger.error(str(error)) # TODO: user message @@ -125,7 +125,7 @@ async def document_diagnostic_with_partial_results( ) try: - async with proxy_utils.find_action_project_and_run_with_partial_results( + async with run_service.find_action_project_and_run_with_partial_results( file_path=file_path, action_name="lint", # TODO: use payload class @@ -191,7 +191,7 @@ async def document_diagnostic_with_partial_results( global_state.progress_reporter( partial_result_token, related_doc_diagnostics ) - except proxy_utils.ActionRunFailed as error: + except run_service.ActionRunFailed as error: # don't throw error because vscode after a few sequential errors will stop # requesting diagnostics until restart. Show user message instead logger.error(str(error)) # TODO: user message @@ -246,7 +246,7 @@ async def run_workspace_diagnostic_with_partial_results( assert global_state.progress_reporter is not None try: - async with proxy_utils.run_with_partial_results( + async with run_service.run_with_partial_results( action_name="lint", params=exec_info.request_data, partial_result_token=partial_result_token, @@ -277,7 +277,7 @@ async def run_workspace_diagnostic_with_partial_results( ] ) global_state.progress_reporter(partial_result_token, lsp_subresult) - except proxy_utils.ActionRunFailed as error: + except run_service.ActionRunFailed as error: # don't throw error because vscode after a few sequential errors will stop # requesting diagnostics until restart. Show user message instead logger.error(str(error)) # TODO: user message @@ -285,7 +285,7 @@ async def run_workspace_diagnostic_with_partial_results( async def workspace_diagnostic_with_partial_results( exec_infos: list[LintActionExecInfo], partial_result_token: str | int -): +) -> WorkspaceDiagnosticReport: try: async with asyncio.TaskGroup() as tg: for exec_info in exec_infos: @@ -353,8 +353,7 @@ async def workspace_diagnostic_with_full_result( async def _workspace_diagnostic( params: types.WorkspaceDiagnosticParams, ) -> types.WorkspaceDiagnosticReport | None: - - relevant_projects_paths: list[Path] = proxy_utils.find_all_projects_with_action( + relevant_projects_paths: list[Path] = run_service.find_all_projects_with_action( action_name="lint", ws_context=global_state.ws_context ) exec_info_by_project_dir_path: dict[Path, LintActionExecInfo] = {} @@ -373,8 +372,10 @@ async def _workspace_diagnostic( # check which runners are active and run in them # # assign files to projects - files_by_projects: dict[Path, list[Path]] = project_analyzer.get_files_by_projects( - projects_dirs_paths=relevant_projects_paths + files_by_projects: dict[ + Path, list[Path] + ] = await project_analyzer.get_files_by_projects( + projects_dirs_paths=relevant_projects_paths, ws_context=global_state.ws_context ) for project_dir_path, files_for_runner in files_by_projects.items(): @@ -388,7 +389,6 @@ async def _workspace_diagnostic( exec_info = exec_info_by_project_dir_path[project_dir_path] if exec_info.action_name == "lint": - # TODO: use payload class exec_info.request_data = { "file_paths": [file_path.as_posix() for file_path in files_for_runner], } @@ -418,10 +418,13 @@ async def workspace_diagnostic( # - pygls will cut information about exception in logs and it will be hard to # understand it try: - return await _workspace_diagnostic(params) + result = await _workspace_diagnostic(params) except Exception as exception: # TODO: user message logger.exception(exception) # lsprotocol allows None as return value, but then vscode throws error # 'cannot read items of null'. keep empty report instead return types.WorkspaceDiagnosticReport(items=[]) + + logger.trace(f"Workspace diagnostic ended: {params}") + return result diff --git a/src/finecode/lsp_server/endpoints/document_sync.py b/src/finecode/lsp_server/endpoints/document_sync.py index 613b61e..56fd933 100644 --- a/src/finecode/lsp_server/endpoints/document_sync.py +++ b/src/finecode/lsp_server/endpoints/document_sync.py @@ -34,7 +34,11 @@ async def document_did_open( try: async with asyncio.TaskGroup() as tg: for project_path in projects_paths: - runners_by_env = global_state.ws_context.ws_projects_extension_runners.get(project_path, {}) + runners_by_env = ( + global_state.ws_context.ws_projects_extension_runners.get( + project_path, {} + ) + ) for runner in runners_by_env.values(): tg.create_task( runner_client.notify_document_did_open( @@ -98,6 +102,6 @@ async def document_did_save( async def document_did_change( ls: LanguageServer, params: types.DidChangeTextDocumentParams ): - global_state.ws_context.opened_documents[params.text_document.uri].version = ( - params.text_document.version - ) + global_state.ws_context.opened_documents[ + params.text_document.uri + ].version = params.text_document.version diff --git a/src/finecode/lsp_server/endpoints/formatting.py b/src/finecode/lsp_server/endpoints/formatting.py index b6c2fee..51d003c 100644 --- a/src/finecode/lsp_server/endpoints/formatting.py +++ b/src/finecode/lsp_server/endpoints/formatting.py @@ -5,7 +5,8 @@ from loguru import logger from lsprotocol import types -from finecode import proxy_utils, pygls_types_utils +from finecode import pygls_types_utils +from finecode.services import run_service from finecode.lsp_server import global_state if TYPE_CHECKING: @@ -18,10 +19,8 @@ async def format_document(ls: LanguageServer, params: types.DocumentFormattingPa file_path = pygls_types_utils.uri_str_to_path(params.text_document.uri) - # first check 'format' action, because it always replaces the whole content, then - # TEXT_DOCUMENT_FORMATTING feature, it can replace also parts of document try: - response = await proxy_utils.find_action_project_and_run( + response = await run_service.find_action_project_and_run( file_path=file_path, action_name="format", params={"file_paths": [file_path], "save": False}, @@ -52,28 +51,6 @@ async def format_document(ls: LanguageServer, params: types.DocumentFormattingPa ) ] - # TODO: restore - # try: - # response = await proxy_utils.find_project_and_run_in_runner( - # file_path=file_path, - # method=types.TEXT_DOCUMENT_FORMATTING, - # params=params, - # response_type=list, # TODO? - # ws_context=global_state.ws_context, - # ) - # except Exception as error: # TODO - # logger.error(f"Error document formatting {file_path}: {error}") - # return None - - # if response is not None and len(response) > 0: - # text_edit = response[0] - # assert isinstance(text_edit, types.TextEdit) - # if text_edit.range.end.character == -1 and text_edit.range.end.line == -1: - # text_edit.range.end = types.Position( - # line=len(doc.lines), - # character=len(doc.lines[-1]) - # ) - return [] diff --git a/src/finecode/lsp_server/endpoints/inlay_hints.py b/src/finecode/lsp_server/endpoints/inlay_hints.py index 8fa3a95..a8a4cf5 100644 --- a/src/finecode/lsp_server/endpoints/inlay_hints.py +++ b/src/finecode/lsp_server/endpoints/inlay_hints.py @@ -5,7 +5,8 @@ from loguru import logger from lsprotocol import types -from finecode import find_project, proxy_utils, pygls_types_utils +from finecode import find_project, pygls_types_utils +from finecode.services import run_service from finecode.lsp_server import global_state if TYPE_CHECKING: @@ -48,7 +49,7 @@ async def document_inlay_hint( logger.trace(f"Document inlay hints requested: {params}") file_path = pygls_types_utils.uri_str_to_path(params.text_document.uri) try: - response = await proxy_utils.find_action_project_and_run( + response = await run_service.find_action_project_and_run( file_path=file_path, action_name="text_document_inlay_hint", params=inlay_hint_params_to_dict(params), diff --git a/src/finecode/lsp_server/lsp_server.py b/src/finecode/lsp_server/lsp_server.py index 751e255..5a16f1c 100644 --- a/src/finecode/lsp_server/lsp_server.py +++ b/src/finecode/lsp_server/lsp_server.py @@ -8,6 +8,7 @@ from pygls.lsp.server import LanguageServer from finecode import services as wm_services +from finecode.runner import manager as runner_manager from finecode.lsp_server import global_state, schemas, services from finecode.lsp_server.endpoints import action_tree as action_tree_endpoints from finecode.lsp_server.endpoints import code_actions as code_actions_endpoints @@ -22,7 +23,9 @@ def create_lsp_server() -> LanguageServer: # handle all requests explicitly because there are different types of requests: # project-specific, workspace-wide. Some Workspace-wide support partial responses, # some not. - server = LanguageServer("FineCode_Workspace_Manager_Server", "v1") + server = LanguageServer( + "FineCode_Workspace_Manager_Server", "v1" + ) register_initialized_feature = server.feature(types.INITIALIZED) register_initialized_feature(_on_initialized) @@ -256,7 +259,7 @@ async def restart_extension_runner(ls: LanguageServer, params): runner_working_dir_str = params_dict["projectPath"] runner_working_dir_path = Path(runner_working_dir_str) - await wm_services.restart_extension_runners( + await runner_manager.restart_extension_runners( runner_working_dir_path, global_state.ws_context ) diff --git a/src/finecode/lsp_server/services.py b/src/finecode/lsp_server/services.py index 2cfb196..8533fa3 100644 --- a/src/finecode/lsp_server/services.py +++ b/src/finecode/lsp_server/services.py @@ -3,7 +3,7 @@ from loguru import logger from finecode import domain, user_messages -from finecode.config import collect_actions, config_models, read_configs +from finecode.config import read_configs from finecode.lsp_server import global_state, schemas from finecode.runner import manager as runner_manager @@ -89,7 +89,6 @@ async def delete_workspace_dir( # find all projects affected by removing of this ws dir project_dir_pathes = global_state.ws_context.ws_projects.keys() - projects_to_remove: list[Path] = [] for project_dir_path in project_dir_pathes: if not project_dir_path.is_relative_to(ws_dir_path_to_remove): continue diff --git a/src/finecode/project_analyzer.py b/src/finecode/project_analyzer.py index 5718062..4578270 100644 --- a/src/finecode/project_analyzer.py +++ b/src/finecode/project_analyzer.py @@ -1,133 +1,55 @@ from pathlib import Path - -def get_files_by_projects(projects_dirs_paths: list[Path]) -> dict[Path, list[Path]]: - files_by_projects_dirs: dict[Path, list[Path]] = {} - - if len(projects_dirs_paths) == 1: - project_dir = projects_dirs_paths[0] - files_by_projects_dirs[project_dir] = [ - path - for path in project_dir.rglob("*.py") - # TODO: make configurable? - if "__testdata__" not in path.relative_to(project_dir).parts - and ".venvs" not in path.relative_to(project_dir).parts - ] - else: - # copy to avoid modifying of argument values - projects_dirs = projects_dirs_paths.copy() - # sort by depth so that child items are first - # default reverse path sorting works so, that child items are before their - # parents - projects_dirs.sort(reverse=True) - for index, project_dir_path in enumerate(projects_dirs): - files_by_projects_dirs[project_dir_path] = [] - - child_project_by_rel_path: dict[Path, Path] = {} - # find children - for current_project_dir_path in projects_dirs[:index]: - if not current_project_dir_path.is_relative_to(project_dir_path): - break - else: - rel_to_project_dir_path = current_project_dir_path.relative_to( - project_dir_path - ) - child_project_by_rel_path[rel_to_project_dir_path] = ( - current_project_dir_path - ) - - # convert child_project_by_rel_path to tree to be able to check whether - # directory contains subrojects without reiterating - child_project_tree: dict[str, str] = {} - for child_rel_path in child_project_by_rel_path.keys(): - current_tree_branch = child_project_tree - for part in child_rel_path.parts: - if part not in current_tree_branch: - current_tree_branch[part] = {} - current_tree_branch = current_tree_branch[part] - - # use set, because one dir item can have multiple subprojects and we need - # it only once - dir_items_with_children: set[str] = set( - [ - dir_item_path.parts[0] - for dir_item_path in child_project_by_rel_path.keys() - ] +from finecode import context +from finecode.services import run_service + + +class FailedToGetProjectFiles(Exception): + def __init__(self, message: str) -> None: + self.message = message + + +async def get_files_by_projects( + projects_dirs_paths: list[Path], ws_context: context.WorkspaceContext +) -> dict[Path, list[Path]]: + files_by_project_dir: dict[Path, list[Path]] = {} + actions_by_project = { + project_dir_path: ["list_project_files_by_lang"] + for project_dir_path in projects_dirs_paths + } + action_payload = {} + + try: + results_by_project = await run_service.run_actions_in_projects( + actions_by_project=actions_by_project, + action_payload=action_payload, + ws_context=ws_context, + concurrently=False, + result_format=run_service.RunResultFormat.JSON, + ) + except run_service.ActionRunFailed as exception: + # TODO: handle it overall + raise FailedToGetProjectFiles(exception.message) + + for project_dir_path, action_results in results_by_project.items(): + list_project_files_action_result = action_results["list_project_files_by_lang"] + if list_project_files_action_result.return_code != 0: + raise FailedToGetProjectFiles( + f"'list_project_files_by_lang' action ended in {project_dir_path} with return code {list_project_files_action_result.return_code}: {list_project_files_action_result.result}" ) - if len(dir_items_with_children) == 0: - # if there are no children with subprojects, we can just rglob - files_by_projects_dirs[project_dir_path].extend( - path - for path in project_dir_path.rglob("*.py") - # TODO: make configurable? - if "__testdata__" not in path.relative_to(project_dir_path).parts - and ".venvs" not in path.relative_to(project_dir_path).parts - ) - else: - # process all dir items which don't have child projects - for dir_item in project_dir_path.iterdir(): - if dir_item.name in dir_items_with_children: - continue - else: - if dir_item.suffix == ".py": - files_by_projects_dirs[project_dir_path].append(dir_item) - elif dir_item.is_dir(): - files_by_projects_dirs[project_dir_path].extend( - path - for path in dir_item.rglob("*.py") - # TODO: make configurable? - if "__testdata__" - not in path.relative_to(project_dir_path).parts - and ".venvs" - not in path.relative_to(project_dir_path).parts - ) - - # process all dir items which have child projects - # - # avoid repeating processing of the same directories which would cause - # duplicates in list of files by saving processed branches - processed_branches: list[dict[str, str]] = [] - for rel_path in child_project_by_rel_path.keys(): - rel_path_parts = rel_path.parts - current_tree_branch = child_project_tree - if current_tree_branch in processed_branches: - continue - processed_branches.append(current_tree_branch) - # iterate from second item because the first one is directory we - # currently processing - for index in range(len(rel_path_parts[1:])): - current_path = project_dir_path / "/".join( - rel_path_parts[: index + 1] - ) - current_tree_branch = current_tree_branch[rel_path_parts[index]] - if current_tree_branch in processed_branches: - continue - processed_branches.append(current_tree_branch) - - for dir_item in current_path.iterdir(): - if dir_item.suffix == ".py": - files_by_projects_dirs[project_dir_path].append( - dir_item - ) - elif dir_item.is_dir(): - if dir_item.name in current_tree_branch: - # it's a path to child project, skip it - continue - else: - # subdirectory without child projects, rglob it - files_by_projects_dirs[project_dir_path].extend( - path - for path in dir_item.rglob("*.py") - # TODO: make configurable? - if "__testdata__" - not in path.relative_to(project_dir_path).parts - and ".venvs" - not in path.relative_to(project_dir_path).parts - ) + project_files_by_lang = list_project_files_action_result.result + files_by_project_dir[project_dir_path] = [ + Path(file_path) + for file_path in project_files_by_lang["files_by_lang"].get("python", []) + ] - return files_by_projects_dirs + return files_by_project_dir -def get_project_files(project_dir_path: Path) -> list[Path]: - files_by_projects = get_files_by_projects([project_dir_path]) +async def get_project_files( + project_dir_path: Path, ws_context: context.WorkspaceContext +) -> list[Path]: + files_by_projects = await get_files_by_projects( + [project_dir_path], ws_context=ws_context + ) return files_by_projects[project_dir_path] diff --git a/src/finecode/runner/manager.py b/src/finecode/runner/manager.py index 117110e..975d560 100644 --- a/src/finecode/runner/manager.py +++ b/src/finecode/runner/manager.py @@ -173,7 +173,7 @@ async def stop_extension_runner(runner: runner_info.ExtensionRunnerInfo) -> None logger.debug("Sent exit to server") await runner.client.stop() logger.trace( - f"Stop extension runner {runner.process_id}" f" in {runner.readable_id}" + f"Stop extension runner {runner.process_id} in {runner.readable_id}" ) else: logger.trace("Extension runner was not running") @@ -194,7 +194,7 @@ def stop_extension_runner_sync(runner: runner_info.ExtensionRunnerInfo) -> None: runner_client.exit_sync(runner) logger.debug("Sent exit to server") logger.trace( - f"Stop extension runner {runner.process_id}" f" in {runner.readable_id}" + f"Stop extension runner {runner.process_id} in {runner.readable_id}" ) else: logger.trace("Extension runner was not running") @@ -238,7 +238,7 @@ async def start_runners_with_presets( raise RunnerFailedToStart( "Failed to initialize runner(s). See previous logs for more details" ) - + for project in projects: try: await read_configs.read_project_config( @@ -253,20 +253,28 @@ async def start_runners_with_presets( ) -async def get_or_start_runners_with_presets(project_dir_path: Path, ws_context: context.WorkspaceContext) -> runner_info.ExtensionRunnerInfo: +async def get_or_start_runners_with_presets( + project_dir_path: Path, ws_context: context.WorkspaceContext +) -> runner_info.ExtensionRunnerInfo: # project is expected to have status `ProjectStatus.CONFIG_VALID` - has_dev_workspace_runner = 'dev_workspace' in ws_context.ws_projects_extension_runners[project_dir_path] + has_dev_workspace_runner = ( + "dev_workspace" in ws_context.ws_projects_extension_runners[project_dir_path] + ) if not has_dev_workspace_runner: project = ws_context.ws_projects[project_dir_path] await start_runners_with_presets([project], ws_context) - dev_workspace_runner = ws_context.ws_projects_extension_runners[project_dir_path]['dev_workspace'] + dev_workspace_runner = ws_context.ws_projects_extension_runners[project_dir_path][ + "dev_workspace" + ] if dev_workspace_runner.status == runner_info.RunnerStatus.RUNNING: return dev_workspace_runner elif dev_workspace_runner.status == runner_info.RunnerStatus.INITIALIZING: await dev_workspace_runner.initialized_event.wait() return dev_workspace_runner else: - raise RunnerFailedToStart(f'Status of dev_workspace runner: {dev_workspace_runner.status}, logs: {dev_workspace_runner.logs_path}') + raise RunnerFailedToStart( + f"Status of dev_workspace runner: {dev_workspace_runner.status}, logs: {dev_workspace_runner.logs_path}" + ) async def start_runner( @@ -401,8 +409,7 @@ async def update_runner_config( ) logger.debug( - f"Updated config of runner {runner.readable_id}," - f" process id {runner.process_id}" + f"Updated config of runner {runner.readable_id}, process id {runner.process_id}" ) @@ -506,3 +513,26 @@ def remove_runner_venv(runner_dir: Path, env_name: str) -> None: if venv_dir_path.exists(): logger.debug(f"Remove venv {venv_dir_path}") shutil.rmtree(venv_dir_path) + + +async def restart_extension_runners( + runner_working_dir_path: Path, ws_context: context.WorkspaceContext +) -> None: + # TODO: reload config? + try: + runners_by_env = ws_context.ws_projects_extension_runners[ + runner_working_dir_path + ] + except KeyError: + logger.error(f"Cannot find runner for {runner_working_dir_path}") + return + + for runner in runners_by_env.values(): + await stop_extension_runner(runner) + + project_def = ws_context.ws_projects[runner.working_dir_path] + await start_runner( + project_def=project_def, + env_name=runner.env_name, + ws_context=ws_context, + ) diff --git a/src/finecode/runner/runner_client.py b/src/finecode/runner/runner_client.py index 575e386..a81d791 100644 --- a/src/finecode/runner/runner_client.py +++ b/src/finecode/runner/runner_client.py @@ -70,8 +70,7 @@ async def send_request( logger.error(f"Extension runner crashed: {error}") await log_process_log_streams(process=runner.client._server) raise NoResponse( - f"Extension runner {runner.readable_id} crashed," - f" no response on {method}" + f"Extension runner {runner.readable_id} crashed, no response on {method}" ) except TimeoutError: raise ResponseTimeout( @@ -103,8 +102,7 @@ def send_request_sync( except RuntimeError as error: logger.error(f"Extension runner crashed? {error}") raise NoResponse( - f"Extension runner {runner.readable_id} crashed," - f" no response on {method}" + f"Extension runner {runner.readable_id} crashed, no response on {method}" ) except TimeoutError: if runner.client._server.returncode is not None: @@ -139,7 +137,7 @@ async def initialize( client_info=types.ClientInfo(name=client_name, version=client_version), trace=types.TraceValue.Verbose, ), - timeout=20 + timeout=20, ) @@ -248,7 +246,9 @@ class RunnerConfig: action_handler_configs: dict[str, dict[str, Any]] -async def update_config(runner: ExtensionRunnerInfo, project_def_path: pathlib.Path, config: RunnerConfig) -> None: +async def update_config( + runner: ExtensionRunnerInfo, project_def_path: pathlib.Path, config: RunnerConfig +) -> None: await send_request( runner=runner, method=types.WORKSPACE_EXECUTE_COMMAND, diff --git a/src/finecode/services.py b/src/finecode/services.py deleted file mode 100644 index 4ed6ffb..0000000 --- a/src/finecode/services.py +++ /dev/null @@ -1,168 +0,0 @@ -import pathlib -import typing - -import ordered_set -from loguru import logger - -from finecode import context, domain, payload_preprocessor, user_messages -from finecode.runner import manager as runner_manager -from finecode.runner import runner_client, runner_info - - -async def restart_extension_runners( - runner_working_dir_path: pathlib.Path, ws_context: context.WorkspaceContext -) -> None: - # TODO: reload config? - try: - runners_by_env = ws_context.ws_projects_extension_runners[ - runner_working_dir_path - ] - except KeyError: - logger.error(f"Cannot find runner for {runner_working_dir_path}") - return - - for runner in runners_by_env.values(): - await runner_manager.stop_extension_runner(runner) - - project_def = ws_context.ws_projects[runner.working_dir_path] - new_runner = await runner_manager.start_runner( - project_def=project_def, - env_name=runner.env_name, - ws_context=ws_context, - ) - - -def on_shutdown(ws_context: context.WorkspaceContext): - - running_runners = [] - for runners_by_env in ws_context.ws_projects_extension_runners.values(): - for runner in runners_by_env.values(): - if runner.status == runner_info.RunnerStatus.RUNNING: - running_runners.append(runner) - - logger.trace(f"Stop all {len(running_runners)} running extension runners") - - for runner in running_runners: - runner_manager.stop_extension_runner_sync(runner=runner) - - # TODO: stop MCP if running - - -class ActionRunFailed(Exception): - def __init__(self, message: str) -> None: - self.message = message - - -RunResultFormat = runner_client.RunResultFormat -RunActionResponse = runner_client.RunActionResponse - - -async def run_action( - action_name: str, - params: dict[str, typing.Any], - project_def: domain.Project, - ws_context: context.WorkspaceContext, - result_format: RunResultFormat = RunResultFormat.JSON, - preprocess_payload: bool = True, -) -> RunActionResponse: - formatted_params = str(params) - if len(formatted_params) > 100: - formatted_params = f"{formatted_params[:100]}..." - logger.trace(f"Execute action {action_name} with {formatted_params}") - - if project_def.status != domain.ProjectStatus.CONFIG_VALID: - raise ActionRunFailed( - f"Project {project_def.dir_path} has no valid configuration and finecode." - " Please check logs." - ) - - if preprocess_payload: - payload = payload_preprocessor.preprocess_for_project( - action_name=action_name, - payload=params, - project_dir_path=project_def.dir_path, - ws_context=ws_context, - ) - else: - payload = params - - # cases: - # - base: all action handlers are in one env - # -> send `run_action` request to runner in env and let it handle concurrency etc. - # It could be done also in workspace manager, but handlers share run context - # - mixed envs: action handlers are in different envs - # -- concurrent execution of handlers - # -- sequential execution of handlers - assert project_def.actions is not None - action = next( - action for action in project_def.actions if action.name == action_name - ) - all_handlers_envs = ordered_set.OrderedSet( - [handler.env for handler in action.handlers] - ) - all_handlers_are_in_one_env = len(all_handlers_envs) == 1 - - if all_handlers_are_in_one_env: - env_name = all_handlers_envs[0] - response = await _run_action_in_env_runner( - action_name=action_name, - payload=payload, - env_name=env_name, - project_def=project_def, - ws_context=ws_context, - result_format=result_format, - ) - else: - # TODO: concurrent vs sequential, this value should be taken from action config - run_concurrently = False # action_name == 'lint' - if run_concurrently: - ... - raise NotImplementedError() - else: - for handler in action.handlers: - # TODO: manage run context - response = await _run_action_in_env_runner( - action_name=action_name, - payload=payload, - env_name=handler.env, - project_def=project_def, - ws_context=ws_context, - result_format=result_format, - ) - - return response - - -async def _run_action_in_env_runner( - action_name: str, - payload: dict[str, typing.Any], - env_name: str, - project_def: domain.Project, - ws_context: context.WorkspaceContext, - result_format: RunResultFormat = RunResultFormat.JSON, -): - try: - runner = await runner_manager.get_or_start_runner( - project_def=project_def, env_name=env_name, ws_context=ws_context - ) - except runner_manager.RunnerFailedToStart as exception: - raise ActionRunFailed( - f"Runner {env_name} in project {project_def.dir_path} failed: {exception.message}" - ) - - try: - response = await runner_client.run_action( - runner=runner, - action_name=action_name, - params=payload, - options={"result_format": result_format}, - ) - except runner_client.BaseRunnerRequestException as error: - await user_messages.error( - f"Action {action_name} failed in {runner.readable_id}: {error.message} . Log file: {runner.logs_path}" - ) - raise ActionRunFailed( - f"Action {action_name} failed in {runner.readable_id}: {error.message} . Log file: {runner.logs_path}" - ) - - return response diff --git a/src/finecode/services/__init__.py b/src/finecode/services/__init__.py new file mode 100644 index 0000000..4bad35c --- /dev/null +++ b/src/finecode/services/__init__.py @@ -0,0 +1,3 @@ +from .shutdown import on_shutdown + +__all__ = ["on_shutdown"] diff --git a/src/finecode/services/run_service/__init__.py b/src/finecode/services/run_service/__init__.py new file mode 100644 index 0000000..a31ae23 --- /dev/null +++ b/src/finecode/services/run_service/__init__.py @@ -0,0 +1,12 @@ +from .exceptions import ActionRunFailed +from .proxy_utils import ( + run_action, + find_action_project_and_run, + find_action_project_and_run_with_partial_results, + find_projects_with_actions, + find_all_projects_with_action, + run_with_partial_results, + start_required_environments, + run_actions_in_projects, + RunResultFormat, +) diff --git a/src/finecode/services/run_service/exceptions.py b/src/finecode/services/run_service/exceptions.py new file mode 100644 index 0000000..675c197 --- /dev/null +++ b/src/finecode/services/run_service/exceptions.py @@ -0,0 +1,8 @@ +class ActionRunFailed(Exception): + def __init__(self, message: str) -> None: + self.message = message + + +class StartingEnvironmentsFailed(Exception): + def __init__(self, message: str) -> None: + self.message = message diff --git a/src/finecode/payload_preprocessor.py b/src/finecode/services/run_service/payload_preprocessor.py similarity index 92% rename from src/finecode/payload_preprocessor.py rename to src/finecode/services/run_service/payload_preprocessor.py index a786a62..9eeffef 100644 --- a/src/finecode/payload_preprocessor.py +++ b/src/finecode/services/run_service/payload_preprocessor.py @@ -4,7 +4,7 @@ from finecode import context, project_analyzer -def preprocess_for_project( +async def preprocess_for_project( action_name: str, payload: dict[str, typing.Any], project_dir_path: pathlib.Path, @@ -53,8 +53,8 @@ def preprocess_for_project( for param, value in processed_payload.items(): if param == "file_paths" and value is None: - processed_payload["file_paths"] = project_analyzer.get_project_files( - project_dir_path + processed_payload["file_paths"] = await project_analyzer.get_project_files( + project_dir_path, ws_context=ws_context ) return processed_payload diff --git a/src/finecode/proxy_utils.py b/src/finecode/services/run_service/proxy_utils.py similarity index 54% rename from src/finecode/proxy_utils.py rename to src/finecode/services/run_service/proxy_utils.py index db595d6..a208bf4 100644 --- a/src/finecode/proxy_utils.py +++ b/src/finecode/services/run_service/proxy_utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import collections.abc import contextlib @@ -7,11 +9,14 @@ import ordered_set from loguru import logger -from finecode import context, domain, find_project, services +from finecode import context, domain, find_project, user_messages from finecode.runner import manager as runner_manager from finecode.runner import runner_client, runner_info from finecode.runner.manager import RunnerFailedToStart -from finecode.services import ActionRunFailed +from finecode.runner.runner_client import RunResultFormat # reexport + +from finecode.services.run_service import payload_preprocessor +from .exceptions import ActionRunFailed, StartingEnvironmentsFailed async def find_action_project( @@ -57,14 +62,14 @@ async def find_action_project_and_run( project = ws_context.ws_projects[project_path] try: - response = await services.run_action( + response = await run_action( action_name=action_name, params=params, project_def=project, ws_context=ws_context, preprocess_payload=False, ) - except services.ActionRunFailed as exception: + except ActionRunFailed as exception: raise exception return response @@ -87,7 +92,7 @@ async def run_action_in_runner( return response -class AsyncList[T](): +class AsyncList[T]: def __init__(self) -> None: self.data: list[T] = [] self.change_event: asyncio.Event = asyncio.Event() @@ -183,11 +188,10 @@ async def run_with_partial_results( result_list=result, partial_result_token=partial_result_token ) ) - action = next(action for action in project.actions if action.name == "lint") + action = next(action for action in project.actions if action.name == action_name) action_envs = ordered_set.OrderedSet( [handler.env for handler in action.handlers] ) - runners_by_env = ws_context.ws_projects_extension_runners[project_dir_path] for env_name in action_envs: try: runner = await runner_manager.get_or_start_runner( @@ -211,10 +215,17 @@ async def run_with_partial_results( yield result except ExceptionGroup as eg: - for exc in eg.exceptions: - logger.exception(exc) + errors: list[str] = [] + for exception in eg.exceptions: + if isinstance(exception, ActionRunFailed): + errors.append(exception.message) + else: + errors.append(str(exception)) + logger.error("Unexpected exception:") + logger.exception(exception) + errors_str = ", ".join(errors) raise ActionRunFailed( - f"Run of {action_name} in {project.dir_path} failed. See logs for more details" + f"Run of {action_name} in {project.dir_path} failed: {errors_str}. See logs for more details" ) @@ -270,11 +281,6 @@ def find_all_projects_with_action( return relevant_projects_paths -class StartingEnvironmentsFailed(Exception): - def __init__(self, message: str) -> None: - self.message = message - - async def start_required_environments( actions_by_projects: dict[pathlib.Path, list[str]], ws_context: context.WorkspaceContext, @@ -339,12 +345,247 @@ async def start_required_environments( ) +async def run_actions_in_running_project( + actions: list[str], + action_payload: dict[str, str], + project: domain.Project, + ws_context: context.WorkspaceContext, + concurrently: bool, + result_format: RunResultFormat, +) -> dict[str, RunActionResponse]: + result_by_action: dict[str, RunActionResponse] = {} + + if concurrently: + run_tasks: list[asyncio.Task] = [] + try: + async with asyncio.TaskGroup() as tg: + for action_name in actions: + run_task = tg.create_task( + run_action( + action_name=action_name, + params=action_payload, + project_def=project, + ws_context=ws_context, + result_format=result_format, + ) + ) + run_tasks.append(run_task) + except ExceptionGroup as eg: + for exception in eg.exceptions: + if isinstance(exception, ActionRunFailed): + logger.error(f"{exception.message} in {project.name}") + else: + logger.error("Unexpected exception:") + logger.exception(exception) + raise ActionRunFailed(f"Running of actions {actions} failed") + + for idx, run_task in enumerate(run_tasks): + run_result = run_task.result() + action_name = actions[idx] + result_by_action[action_name] = run_result + else: + for action_name in actions: + try: + run_result = await run_action( + action_name=action_name, + params=action_payload, + project_def=project, + ws_context=ws_context, + result_format=result_format, + ) + except ActionRunFailed as exception: + raise ActionRunFailed( + f"Running of action {action_name} failed: {exception.message}" + ) + except Exception as error: + logger.error("Unexpected exception") + logger.exception(error) + raise ActionRunFailed( + f"Running of action {action_name} failed with unexpected exception" + ) + + result_by_action[action_name] = run_result + + return result_by_action + + +async def run_actions_in_projects( + actions_by_project: dict[pathlib.Path, list[str]], + action_payload: dict[str, str], + ws_context: context.WorkspaceContext, + concurrently: bool, + result_format: RunResultFormat, +) -> dict[pathlib.Path, dict[str, RunActionResponse]]: + project_handler_tasks: list[asyncio.Task] = [] + try: + async with asyncio.TaskGroup() as tg: + for project_dir_path, actions_to_run in actions_by_project.items(): + project = ws_context.ws_projects[project_dir_path] + project_task = tg.create_task( + run_actions_in_running_project( + actions=actions_to_run, + action_payload=action_payload, + project=project, + ws_context=ws_context, + concurrently=concurrently, + result_format=result_format, + ) + ) + project_handler_tasks.append(project_task) + except ExceptionGroup as eg: + for exception in eg.exceptions: + # TODO: merge all in one? + raise exception + + results = {} + projects_paths = list(actions_by_project.keys()) + for idx, project_task in enumerate(project_handler_tasks): + project_dir_path = projects_paths[idx] + results[project_dir_path] = project_task.result() + + return results + + +def find_projects_with_actions( + ws_context: context.WorkspaceContext, actions: list[str] +) -> dict[pathlib.Path, list[str]]: + actions_by_project: dict[pathlib.Path, list[str]] = {} + actions_set = ordered_set.OrderedSet(actions) + + for project in ws_context.ws_projects.values(): + project_actions_names = [action.name for action in project.actions] + # find which of requested actions are available in the project + action_to_run_in_project = actions_set & ordered_set.OrderedSet( + project_actions_names + ) + relevant_actions_in_project = list(action_to_run_in_project) + if len(relevant_actions_in_project) > 0: + actions_by_project[project.dir_path] = relevant_actions_in_project + + return actions_by_project + + +RunResultFormat = runner_client.RunResultFormat +RunActionResponse = runner_client.RunActionResponse + + +async def run_action( + action_name: str, + params: dict[str, typing.Any], + project_def: domain.Project, + ws_context: context.WorkspaceContext, + result_format: RunResultFormat = RunResultFormat.JSON, + preprocess_payload: bool = True, +) -> RunActionResponse: + formatted_params = str(params) + if len(formatted_params) > 100: + formatted_params = f"{formatted_params[:100]}..." + logger.trace(f"Execute action {action_name} with {formatted_params}") + + if project_def.status != domain.ProjectStatus.CONFIG_VALID: + raise ActionRunFailed( + f"Project {project_def.dir_path} has no valid configuration and finecode." + " Please check logs." + ) + + if preprocess_payload: + payload = await payload_preprocessor.preprocess_for_project( + action_name=action_name, + payload=params, + project_dir_path=project_def.dir_path, + ws_context=ws_context, + ) + else: + payload = params + + # cases: + # - base: all action handlers are in one env + # -> send `run_action` request to runner in env and let it handle concurrency etc. + # It could be done also in workspace manager, but handlers share run context + # - mixed envs: action handlers are in different envs + # -- concurrent execution of handlers + # -- sequential execution of handlers + assert project_def.actions is not None + action = next( + action for action in project_def.actions if action.name == action_name + ) + all_handlers_envs = ordered_set.OrderedSet( + [handler.env for handler in action.handlers] + ) + all_handlers_are_in_one_env = len(all_handlers_envs) == 1 + + if all_handlers_are_in_one_env: + env_name = all_handlers_envs[0] + response = await _run_action_in_env_runner( + action_name=action_name, + payload=payload, + env_name=env_name, + project_def=project_def, + ws_context=ws_context, + result_format=result_format, + ) + else: + # TODO: concurrent vs sequential, this value should be taken from action config + run_concurrently = False # action_name == 'lint' + if run_concurrently: + ... + raise NotImplementedError() + else: + for handler in action.handlers: + # TODO: manage run context + response = await _run_action_in_env_runner( + action_name=action_name, + payload=payload, + env_name=handler.env, + project_def=project_def, + ws_context=ws_context, + result_format=result_format, + ) + + return response + + +async def _run_action_in_env_runner( + action_name: str, + payload: dict[str, typing.Any], + env_name: str, + project_def: domain.Project, + ws_context: context.WorkspaceContext, + result_format: RunResultFormat = RunResultFormat.JSON, +): + try: + runner = await runner_manager.get_or_start_runner( + project_def=project_def, env_name=env_name, ws_context=ws_context + ) + except runner_manager.RunnerFailedToStart as exception: + raise ActionRunFailed( + f"Runner {env_name} in project {project_def.dir_path} failed: {exception.message}" + ) + + try: + response = await runner_client.run_action( + runner=runner, + action_name=action_name, + params=payload, + options={"result_format": result_format}, + ) + except runner_client.BaseRunnerRequestException as error: + await user_messages.error( + f"Action {action_name} failed in {runner.readable_id}: {error.message} . Log file: {runner.logs_path}" + ) + raise ActionRunFailed( + f"Action {action_name} failed in {runner.readable_id}: {error.message} . Log file: {runner.logs_path}" + ) + + return response + + __all__ = [ "find_action_project_and_run", "find_action_project_and_run_with_partial_results", + "find_projects_with_actions", + "find_all_projects_with_action", "run_with_partial_results", - # reexport for easier use of proxy helpers - "ActionRunFailed", "start_required_environments", - "StartingEnvironmentsFailed", + "run_actions_in_projects", ] diff --git a/src/finecode/services/shutdown.py b/src/finecode/services/shutdown.py new file mode 100644 index 0000000..eda2fa5 --- /dev/null +++ b/src/finecode/services/shutdown.py @@ -0,0 +1,20 @@ +from loguru import logger + +from finecode import context +from finecode.runner import manager as runner_manager +from finecode.runner import runner_info + + +def on_shutdown(ws_context: context.WorkspaceContext): + running_runners = [] + for runners_by_env in ws_context.ws_projects_extension_runners.values(): + for runner in runners_by_env.values(): + if runner.status == runner_info.RunnerStatus.RUNNING: + running_runners.append(runner) + + logger.trace(f"Stop all {len(running_runners)} running extension runners") + + for runner in running_runners: + runner_manager.stop_extension_runner_sync(runner=runner) + + # TODO: stop MCP if running From 194b62a6269ab474d5c9fb46f590a6f4e14fd713 Mon Sep 17 00:00:00 2001 From: Vladyslav Hnatiuk Date: Sun, 14 Sep 2025 09:00:22 +0200 Subject: [PATCH 06/11] Move InvalidProjectConfig exception in interface from implementation --- .../interfaces/iprojectinfoprovider.py | 5 +++++ .../impls/project_info_provider.py | 7 +------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectinfoprovider.py b/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectinfoprovider.py index 87d94ab..f978b3f 100644 --- a/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectinfoprovider.py +++ b/finecode_extension_api/src/finecode_extension_api/interfaces/iprojectinfoprovider.py @@ -14,3 +14,8 @@ async def get_project_raw_config( ) -> dict[str, Any]: ... async def get_current_project_raw_config(self) -> dict[str, Any]: ... + + +class InvalidProjectConfig(Exception): + def __init__(self, message: str) -> None: + self.message = message diff --git a/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py b/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py index 9936972..4c8a469 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py +++ b/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py @@ -4,11 +4,6 @@ from finecode_extension_api.interfaces import iprojectinfoprovider -class InvalidProjectConfig(Exception): - def __init__(self, message: str) -> None: - self.message = message - - class ProjectInfoProvider(iprojectinfoprovider.IProjectInfoProvider): def __init__( self, @@ -29,7 +24,7 @@ async def get_current_project_package_name(self) -> str: project_raw_config = await self.get_current_project_raw_config() raw_name = project_raw_config.get("project", {}).get("name", None) if raw_name is None: - raise InvalidProjectConfig("project.name not found in project config") + raise iprojectinfoprovider.InvalidProjectConfig("project.name not found in project config") return raw_name.replace("-", "_") From fb898c0b083eb528c18029285091d52d690e81cf Mon Sep 17 00:00:00 2001 From: Vladyslav Hnatiuk Date: Sun, 14 Sep 2025 09:03:39 +0200 Subject: [PATCH 07/11] Import shutdown service in WM instead of services package --- src/finecode/cli_app/commands/dump_config_cmd.py | 6 +++--- src/finecode/cli_app/commands/prepare_envs_cmd.py | 6 +++--- src/finecode/cli_app/commands/run_cmd.py | 6 +++--- src/finecode/lsp_server/lsp_server.py | 4 ++-- src/finecode/services/__init__.py | 3 --- src/finecode/services/{shutdown.py => shutdown_service.py} | 0 6 files changed, 11 insertions(+), 14 deletions(-) rename src/finecode/services/{shutdown.py => shutdown_service.py} (100%) diff --git a/src/finecode/cli_app/commands/dump_config_cmd.py b/src/finecode/cli_app/commands/dump_config_cmd.py index d2c6662..281b2a7 100644 --- a/src/finecode/cli_app/commands/dump_config_cmd.py +++ b/src/finecode/cli_app/commands/dump_config_cmd.py @@ -2,8 +2,8 @@ from loguru import logger -from finecode import context, services -from finecode.services import run_service +from finecode import context +from finecode.services import run_service, shutdown_service from finecode.config import config_models, read_configs from finecode.runner import manager as runner_manager @@ -86,4 +86,4 @@ async def dump_config(workdir_path: pathlib.Path, project_name: str): ) logger.info(f"Dumped config into {dump_file_path}") finally: - services.on_shutdown(ws_context) + shutdown_service.on_shutdown(ws_context) diff --git a/src/finecode/cli_app/commands/prepare_envs_cmd.py b/src/finecode/cli_app/commands/prepare_envs_cmd.py index c581a91..87d8f7b 100644 --- a/src/finecode/cli_app/commands/prepare_envs_cmd.py +++ b/src/finecode/cli_app/commands/prepare_envs_cmd.py @@ -3,8 +3,8 @@ from loguru import logger -from finecode import context, domain, services -from finecode.services import run_service +from finecode import context, domain +from finecode.services import run_service, shutdown_service from finecode.cli_app import utils from finecode.config import collect_actions, config_models, read_configs from finecode.runner import manager as runner_manager @@ -138,7 +138,7 @@ async def prepare_envs(workdir_path: pathlib.Path, recreate: bool) -> None: if result_return_code != 0: raise PrepareEnvsFailed(result_output) finally: - services.on_shutdown(ws_context) + shutdown_service.on_shutdown(ws_context) def remove_dev_workspace_envs( diff --git a/src/finecode/cli_app/commands/run_cmd.py b/src/finecode/cli_app/commands/run_cmd.py index a0d725a..e17912b 100644 --- a/src/finecode/cli_app/commands/run_cmd.py +++ b/src/finecode/cli_app/commands/run_cmd.py @@ -6,8 +6,8 @@ import ordered_set from loguru import logger -from finecode import context, domain, services -from finecode.services import run_service +from finecode import context, domain +from finecode.services import run_service, shutdown_service from finecode.config import collect_actions, config_models, read_configs from finecode.runner import manager as runner_manager @@ -165,7 +165,7 @@ async def run_actions( except run_service.ActionRunFailed as exception: raise RunFailed(f"Failed to run actions: {exception.message}") finally: - services.on_shutdown(ws_context) + shutdown_service.on_shutdown(ws_context) def get_projects_by_names( diff --git a/src/finecode/lsp_server/lsp_server.py b/src/finecode/lsp_server/lsp_server.py index 5a16f1c..9d7c1b3 100644 --- a/src/finecode/lsp_server/lsp_server.py +++ b/src/finecode/lsp_server/lsp_server.py @@ -7,7 +7,7 @@ from lsprotocol import types from pygls.lsp.server import LanguageServer -from finecode import services as wm_services +from finecode.services import shutdown_service from finecode.runner import manager as runner_manager from finecode.lsp_server import global_state, schemas, services from finecode.lsp_server.endpoints import action_tree as action_tree_endpoints @@ -242,7 +242,7 @@ async def _workspace_did_change_workspace_folders( def _on_shutdown(ls: LanguageServer, params): logger.info("on shutdown handler", params) - wm_services.on_shutdown(global_state.ws_context) + shutdown_service.on_shutdown(global_state.ws_context) async def reset(ls: LanguageServer, params): diff --git a/src/finecode/services/__init__.py b/src/finecode/services/__init__.py index 4bad35c..e69de29 100644 --- a/src/finecode/services/__init__.py +++ b/src/finecode/services/__init__.py @@ -1,3 +0,0 @@ -from .shutdown import on_shutdown - -__all__ = ["on_shutdown"] diff --git a/src/finecode/services/shutdown.py b/src/finecode/services/shutdown_service.py similarity index 100% rename from src/finecode/services/shutdown.py rename to src/finecode/services/shutdown_service.py From 2c3a6389cdee56967e14045db74c3916083f2088 Mon Sep 17 00:00:00 2001 From: Vladyslav Hnatiuk Date: Sun, 14 Sep 2025 09:11:05 +0200 Subject: [PATCH 08/11] Add python_version in pyrefly config --- .../fine_python_pyrefly/fine_python_pyrefly/lint_handler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py b/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py index 7dbb89c..111ff93 100644 --- a/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py +++ b/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py @@ -12,7 +12,7 @@ @dataclasses.dataclass class PyreflyLintHandlerConfig(code_action.ActionHandlerConfig): - ... + python_version: str | None = None class PyreflyLintHandler( @@ -104,9 +104,11 @@ async def run_pyrefly_lint_on_single_file( "--output-format=json", "--disable-search-path-heuristics=true", "--skip-interpreter-query", - "--python-version='3.11'" # TODO ] + if self.config.python_version is not None: + cmd.append(f"--python-version='{self.config.python_version}'") + for path in site_package_pathes: cmd.append(f'--site-package-path={str(path)}') cmd.append(str(file_path)) From 478b73d48e32b0074d44f3fbeb56e24624752f50 Mon Sep 17 00:00:00 2001 From: Vladyslav Hnatiuk Date: Sun, 14 Sep 2025 13:14:36 +0200 Subject: [PATCH 09/11] fine_python_pip v0.1.1: add editable_mode config parameter --- extensions/fine_python_pip/pyproject.toml | 3 ++- .../src/fine_python_pip/install_deps_in_env_handler.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/extensions/fine_python_pip/pyproject.toml b/extensions/fine_python_pip/pyproject.toml index ed49dd9..00749b3 100644 --- a/extensions/fine_python_pip/pyproject.toml +++ b/extensions/fine_python_pip/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fine_python_pip" -version = "0.1.0" +version = "0.1.1" description = "" authors = [{ name = "Vladyslav Hnatiuk", email = "aders1234@gmail.com" }] readme = "README.md" @@ -17,3 +17,4 @@ presets = [{ source = "finecode_dev_common_preset" }] finecode_dev_common_preset = { path = "../../finecode_dev_common_preset", editable = true } finecode = { path = "../../", editable = true } finecode_extension_runner = { path = "../../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../../finecode_extension_api", editable = true } diff --git a/extensions/fine_python_pip/src/fine_python_pip/install_deps_in_env_handler.py b/extensions/fine_python_pip/src/fine_python_pip/install_deps_in_env_handler.py index 45deff3..40d5bf4 100644 --- a/extensions/fine_python_pip/src/fine_python_pip/install_deps_in_env_handler.py +++ b/extensions/fine_python_pip/src/fine_python_pip/install_deps_in_env_handler.py @@ -11,6 +11,7 @@ @dataclasses.dataclass class PipInstallDepsInEnvHandlerConfig(code_action.ActionHandlerConfig): find_links: list[str] | None = None + editable_mode: str | None = None class PipInstallDepsInEnvHandler( @@ -62,7 +63,10 @@ def _construct_pip_install_cmd( if self.config.find_links is not None: for link in self.config.find_links: - install_params += f' --find-links="{link}" ' + install_params += f'--find-links="{link}" ' + + if self.config.editable_mode is not None: + install_params += f"--config-settings editable_mode='{self.config.editable_mode}' " for dependency in dependencies: if dependency.editable: From 120f411ef5abebad440bdbe161b6d509067c2027 Mon Sep 17 00:00:00 2001 From: Vladyslav Hnatiuk Date: Sun, 14 Sep 2025 18:28:30 +0200 Subject: [PATCH 10/11] Update config in dev_workspace runner after start to provide config with resolved presets. This allows to customize behavior in presets of the most actions executed in dev_workspace. Add current package to runtime env deps. Add 'additional_dirs' config parameter to list_project_files_by_lang_python handler. Fix copying of action subresult. --- .../list_project_files_by_lang_python.py | 27 ++++++++++++--- .../_services/run_action.py | 5 ++- src/finecode/config/read_configs.py | 33 +++++++++++++++++++ src/finecode/runner/manager.py | 9 ++++- 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/extensions/fine_python_package_info/fine_python_package_info/list_project_files_by_lang_python.py b/extensions/fine_python_package_info/fine_python_package_info/list_project_files_by_lang_python.py index 0116201..a279b86 100644 --- a/extensions/fine_python_package_info/fine_python_package_info/list_project_files_by_lang_python.py +++ b/extensions/fine_python_package_info/fine_python_package_info/list_project_files_by_lang_python.py @@ -1,4 +1,4 @@ -from finecode_extension_api.interfaces import iprojectinfoprovider, ipypackagelayoutinfoprovider +from finecode_extension_api.interfaces import iprojectinfoprovider, ipypackagelayoutinfoprovider, ilogger import dataclasses import pathlib @@ -7,8 +7,10 @@ @dataclasses.dataclass -class ListProjectFilesByLangPythonHandlerConfig(code_action.ActionHandlerConfig): ... -# TODO: parameter for additional dirs +class ListProjectFilesByLangPythonHandlerConfig(code_action.ActionHandlerConfig): + # list of relative pathes relative to project directory with additional python + # sources if they are not in one of default pathes + additional_dirs: list[pathlib.Path] | None = None class ListProjectFilesByLangPythonHandler( @@ -16,9 +18,17 @@ class ListProjectFilesByLangPythonHandler( list_project_files_by_lang_action.ListProjectFilesByLangAction, ListProjectFilesByLangPythonHandlerConfig ] ): - def __init__(self, project_info_provider: iprojectinfoprovider.IProjectInfoProvider, py_package_layout_info_provider: ipypackagelayoutinfoprovider.IPyPackageLayoutInfoProvider) -> None: + def __init__( + self, + config: ListProjectFilesByLangPythonHandlerConfig, + project_info_provider: iprojectinfoprovider.IProjectInfoProvider, + py_package_layout_info_provider: ipypackagelayoutinfoprovider.IPyPackageLayoutInfoProvider, + logger: ilogger.ILogger + ) -> None: + self.config = config self.project_info_provider = project_info_provider self.py_package_layout_info_provider = py_package_layout_info_provider + self.logger = logger self.current_project_dir_path = self.project_info_provider.get_current_project_dir_path() self.tests_dir_path = self.current_project_dir_path / 'tests' @@ -42,6 +52,15 @@ async def run( if self.setup_py_path.exists(): py_files.append(self.setup_py_path) + + if self.config.additional_dirs is not None: + for dir_path in self.config.additional_dirs: + dir_absolute_path = self.current_project_dir_path / dir_path + if not dir_absolute_path.exists(): + self.logger.warning(f"Skip {dir_path} because {dir_absolute_path} doesn't exist") + continue + + py_files += list(dir_absolute_path.rglob('*.py')) return list_project_files_by_lang_action.ListProjectFilesByLangRunResult( files_by_lang={"python": py_files} diff --git a/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py b/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py index 7254891..ac34f2e 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py +++ b/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py @@ -556,8 +556,11 @@ async def run_subresult_coros_concurrently( # in it and result from action handler must stay immutable (e.g. it can # reference to cache) action_subresult_type = type(coro_result) + # use pydantic dataclass as constructor because it instantiates classes + # recursively, normal dataclass only on the first level + action_subresult_type_pydantic = pydantic_dataclass(action_subresult_type) action_subresult_dict = dataclasses.asdict(coro_result) - action_subresult = action_subresult_type(**action_subresult_dict) + action_subresult = action_subresult_type_pydantic(**action_subresult_dict) else: action_subresult.update(coro_result) diff --git a/src/finecode/config/read_configs.py b/src/finecode/config/read_configs.py index 7501943..c288948 100644 --- a/src/finecode/config/read_configs.py +++ b/src/finecode/config/read_configs.py @@ -528,6 +528,39 @@ def handler_to_dict(handler: domain.ActionHandler) -> dict[str, str | list[str]] def add_runtime_dependency_group_if_new(project_config: dict[str, Any]) -> None: runtime_dependencies = project_config.get("project", {}).get("dependencies", []) + + # add root package to runtime env if it is not there yet. It is done here and not + # in package installer, because runtime deps group can be included in other groups + # and root package should be installed in them as well + root_package_name = project_config.get("project", {}).get("name", None) + if root_package_name is None: + raise config_models.ConfigurationError("project.name not found in config") + root_package_in_runtime_deps = any( + dep for dep in runtime_dependencies if get_dependency_name(dep) == root_package_name + ) + if not root_package_in_runtime_deps: + runtime_dependencies.insert(0, root_package_name) + + # make editable. Example: + # [tool.finecode.env.runtime.dependencies] + # package_name = { path = "./", editable = true } + if 'tool' not in project_config: + project_config['tool'] = {} + tool_config = project_config['tool'] + if 'finecode' not in tool_config: + tool_config['finecode'] = {} + finecode_config = tool_config['finecode'] + if 'env' not in finecode_config: + finecode_config['env'] = {} + finecode_env_config = finecode_config['env'] + if 'runtime' not in finecode_env_config: + finecode_env_config['runtime'] = {} + runtime_env_config = finecode_env_config['runtime'] + if 'dependencies' not in runtime_env_config: + runtime_env_config['dependencies'] = {} + runtime_env_deps = runtime_env_config['dependencies'] + if root_package_name not in runtime_env_deps: + runtime_env_deps[root_package_name] = { "path": "./", "editable": True} deps_groups = add_or_get_dict_key_value(project_config, "dependency-groups", {}) if "runtime" not in deps_groups: diff --git a/src/finecode/runner/manager.py b/src/finecode/runner/manager.py index 975d560..29b9739 100644 --- a/src/finecode/runner/manager.py +++ b/src/finecode/runner/manager.py @@ -185,7 +185,7 @@ def stop_extension_runner_sync(runner: runner_info.ExtensionRunnerInfo) -> None: logger.debug("Send shutdown to server") try: runner_client.shutdown_sync(runner=runner) - except Exception as e: + except Exception: # currently we get (almost?) always this error. TODO: Investigate why # mute for now to make output less verbose # logger.error(f"Failed to shutdown: {e}") @@ -240,6 +240,9 @@ async def start_runners_with_presets( ) for project in projects: + if project_status != domain.ProjectStatus.CONFIG_VALID: + continue + try: await read_configs.read_project_config( project=project, ws_context=ws_context @@ -251,6 +254,10 @@ async def start_runners_with_presets( raise RunnerFailedToStart( f"Reading project config with presets and collecting actions in {project.dir_path} failed: {exception.message}" ) + + # update config of dev_workspace runner, the new config contains resolved presets + dev_workspace_runner = ws_context.ws_projects_extension_runners[project.dir_path]['dev_workspace'] + await update_runner_config(runner=dev_workspace_runner, project=project) async def get_or_start_runners_with_presets( From 906f4cc96d6a669abe5f3d17567d2b1af9dca759 Mon Sep 17 00:00:00 2001 From: Vladyslav Hnatiuk Date: Mon, 15 Sep 2025 06:33:01 +0200 Subject: [PATCH 11/11] Add error handling in ipypackagelayoutinfoprovider and its implementation, cache package names --- .../py_package_layout_info_provider.py | 50 +++++++++++++++---- .../actions/list_project_files_by_lang.py | 1 - .../ipypackagelayoutinfoprovider.py | 10 ++++ .../finecode_extension_runner/di/bootstrap.py | 3 +- 4 files changed, 53 insertions(+), 11 deletions(-) diff --git a/extensions/fine_python_package_info/fine_python_package_info/py_package_layout_info_provider.py b/extensions/fine_python_package_info/fine_python_package_info/py_package_layout_info_provider.py index c347db7..c74584a 100644 --- a/extensions/fine_python_package_info/fine_python_package_info/py_package_layout_info_provider.py +++ b/extensions/fine_python_package_info/fine_python_package_info/py_package_layout_info_provider.py @@ -3,42 +3,74 @@ import tomlkit -from finecode_extension_api.interfaces import ifilemanager, ipypackagelayoutinfoprovider +from finecode_extension_api.interfaces import ifilemanager, ipypackagelayoutinfoprovider, icache from finecode_extension_api import service +class ConfigParseError(Exception): + def __init__(self, message: str) -> None: + self.message = message + + class PyPackageLayoutInfoProvider(ipypackagelayoutinfoprovider.IPyPackageLayoutInfoProvider, service.Service): - def __init__(self, file_manager: ifilemanager.IFileManager) -> None: + PACKAGE_NAME_CACHE_KEY = 'PyPackageLayoutInfoProviderPackageName' + + def __init__(self, file_manager: ifilemanager.IFileManager, cache: icache.ICache) -> None: self.file_manager = file_manager - # TODO: cache package name by file version? + self.cache = cache async def _get_package_name(self, package_dir_path: pathlib.Path) -> str: + # raises ConfigParseError package_def_file = package_dir_path / 'pyproject.toml' if not package_def_file.exists(): raise NotImplementedError("Only python packages with pyproject.toml config file are supported") + try: + cached_package_name = await self.cache.get_file_cache(file_path=package_def_file, key=self.PACKAGE_NAME_CACHE_KEY) + return cached_package_name + except icache.CacheMissException: + ... + package_def_file_content = await self.file_manager.get_content(file_path=package_def_file) - # TODO: handle errors - package_def_dict = tomlkit.loads(package_def_file_content) + package_def_file_version = await self.file_manager.get_file_version(file_path=package_def_file) + try: + package_def_dict = tomlkit.loads(package_def_file_content) + except tomlkit.exceptions.ParseError as exception: + raise ConfigParseError(f"Failed to parse package config {package_def_file}: {exception.message} at {exception.line}:{exception.col}") + package_raw_name = package_def_dict.get('project', {}).get('name', None) if package_raw_name is None: raise ValueError(f"package.name not found in {package_def_file}") - return package_raw_name.replace('-', '_') + package_name = package_raw_name.replace('-', '_') + await self.cache.save_file_cache(file_path=package_def_file, file_version=package_def_file_version, key=self.PACKAGE_NAME_CACHE_KEY, value=package_name) + return package_name async def get_package_layout(self, package_dir_path: pathlib.Path) -> ipypackagelayoutinfoprovider.PyPackageLayout: if (package_dir_path / 'src').exists(): return ipypackagelayoutinfoprovider.PyPackageLayout.SRC else: - package_name = await self._get_package_name(package_dir_path=package_dir_path) + try: + package_name = await self._get_package_name(package_dir_path=package_dir_path) + except ConfigParseError as exception: + raise ipypackagelayoutinfoprovider.FailedToGetPackageLayout(exception.message) + if (package_dir_path / package_name).exists(): return ipypackagelayoutinfoprovider.PyPackageLayout.FLAT else: return ipypackagelayoutinfoprovider.PyPackageLayout.CUSTOM async def get_package_src_root_dir_path(self, package_dir_path: str) -> pathlib.Path: - package_layout = await self.get_package_layout(package_dir_path=package_dir_path) - package_name = await self._get_package_name(package_dir_path=package_dir_path) + try: + package_layout = await self.get_package_layout(package_dir_path=package_dir_path) + except ipypackagelayoutinfoprovider.FailedToGetPackageLayout as exception: + raise ipypackagelayoutinfoprovider.FailedToGetPackageSrcRootDirPath(exception.message) + + try: + package_name = await self._get_package_name(package_dir_path=package_dir_path) + except ConfigParseError as exception: + raise ipypackagelayoutinfoprovider.FailedToGetPackageSrcRootDirPath(exception.message) + if package_layout == ipypackagelayoutinfoprovider.PyPackageLayout.SRC: return package_dir_path / 'src' / package_name elif package_layout == ipypackagelayoutinfoprovider.PyPackageLayout.FLAT: diff --git a/finecode_extension_api/src/finecode_extension_api/actions/list_project_files_by_lang.py b/finecode_extension_api/src/finecode_extension_api/actions/list_project_files_by_lang.py index dbcf60a..f7f7a80 100644 --- a/finecode_extension_api/src/finecode_extension_api/actions/list_project_files_by_lang.py +++ b/finecode_extension_api/src/finecode_extension_api/actions/list_project_files_by_lang.py @@ -1,7 +1,6 @@ import dataclasses import pathlib import sys -import typing if sys.version_info >= (3, 12): from typing import override diff --git a/finecode_extension_api/src/finecode_extension_api/interfaces/ipypackagelayoutinfoprovider.py b/finecode_extension_api/src/finecode_extension_api/interfaces/ipypackagelayoutinfoprovider.py index a337fdc..c5a7e56 100644 --- a/finecode_extension_api/src/finecode_extension_api/interfaces/ipypackagelayoutinfoprovider.py +++ b/finecode_extension_api/src/finecode_extension_api/interfaces/ipypackagelayoutinfoprovider.py @@ -21,3 +21,13 @@ async def get_package_src_root_dir_path( # if you need path to directory which is added to sys.path during execution, take # parent of this directory. ... + + +class FailedToGetPackageLayout(Exception): + def __init__(self, message: str) -> None: + self.message = message + + +class FailedToGetPackageSrcRootDirPath(Exception): + def __init__(self, message: str) -> None: + self.message = message diff --git a/finecode_extension_runner/src/finecode_extension_runner/di/bootstrap.py b/finecode_extension_runner/src/finecode_extension_runner/di/bootstrap.py index 30e147e..fd011db 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/di/bootstrap.py +++ b/finecode_extension_runner/src/finecode_extension_runner/di/bootstrap.py @@ -170,6 +170,7 @@ async def project_file_classifier_factory( async def py_package_layout_info_provider_factory(container): file_manager = await resolver.get_service_instance(ifilemanager.IFileManager) + cache = await resolver.get_service_instance(icache.ICache) return fine_python_package_info.PyPackageLayoutInfoProvider( - file_manager=file_manager, + file_manager=file_manager, cache=cache )