From cf5dc5af6140e172186af03615d84c38eb6a5da9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 19 Oct 2022 09:10:01 -0400 Subject: [PATCH] Standalone implementation of tool shed client... --- .github/workflows/toolshed.yaml | 5 +- .../repository_dependency_manager.py | 4 +- .../tool_shed/util/dependency_display.py | 21 +- .../galaxy/controllers/admin_toolshed.py | 15 +- lib/tool_shed/test/base/test_db_util.py | 34 +- lib/tool_shed/test/base/twilltestcase.py | 369 ++++++++++++++++-- ..._repository_basic_circular_dependencies.py | 6 +- ...est_1050_circular_dependencies_4_levels.py | 3 +- ...vanced_circular_dependency_installation.py | 21 +- ...est_1090_repository_dependency_handling.py | 4 +- ...ll_repository_with_complex_dependencies.py | 1 - ...80_circular_prior_installation_required.py | 5 +- .../test_1300_reset_all_metadata.py | 13 +- .../functional/test_1460_data_managers.py | 11 +- 14 files changed, 404 insertions(+), 108 deletions(-) diff --git a/.github/workflows/toolshed.yaml b/.github/workflows/toolshed.yaml index 64f148c0f346..e8fa6e852a48 100644 --- a/.github/workflows/toolshed.yaml +++ b/.github/workflows/toolshed.yaml @@ -21,6 +21,7 @@ jobs: strategy: matrix: python-version: ['3.7'] + test-install-client: ['standalone', 'galaxy_api'] services: postgres: image: postgres:13 @@ -54,9 +55,11 @@ jobs: key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-toolshed - name: Run tests run: './run_tests.sh -toolshed' + env: + TOOL_SHED_TEST_INSTALL_CLIENT: ${{ matrix.test-install-client }} working-directory: 'galaxy root' - uses: actions/upload-artifact@v3 if: failure() with: - name: Toolshed test results (${{ matrix.python-version }}) + name: Toolshed test results (${{ matrix.python-version }}, ${{ matrix.test-install-client }}) path: 'galaxy root/run_toolshed_tests.html' diff --git a/lib/galaxy/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py b/lib/galaxy/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py index 9819fe44c40b..bd761028f773 100644 --- a/lib/galaxy/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -15,6 +15,7 @@ urlopen, ) +from galaxy.tool_shed.galaxy_install import installed_repository_manager from galaxy.tool_shed.galaxy_install.tools import tool_panel_manager from galaxy.tool_shed.util import repository_util from galaxy.tool_shed.util.container_util import get_components_from_key @@ -263,7 +264,8 @@ def create_repository_dependency_objects( log.info( f"Reactivating deactivated tool_shed_repository '{str(repository_db_record.name)}'." ) - self.app.installed_repository_manager.activate_repository(repository_db_record) + irm = installed_repository_manager.InstalledRepositoryManager(self.app) + irm.activate_repository(repository_db_record) # No additional updates to the database record are necessary. can_update_db_record = False elif repository_db_record.status not in [ diff --git a/lib/galaxy/tool_shed/util/dependency_display.py b/lib/galaxy/tool_shed/util/dependency_display.py index 1d27431e4956..dc08d9350dcb 100644 --- a/lib/galaxy/tool_shed/util/dependency_display.py +++ b/lib/galaxy/tool_shed/util/dependency_display.py @@ -2,6 +2,7 @@ import os from galaxy import util +from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager from galaxy.tool_shed.util import utility_container_manager from galaxy.util.tool_shed.common_util import parse_repository_dependency_tuple @@ -197,11 +198,12 @@ def populate_containers_dict_from_repository_metadata(self, repository): """ metadata = repository.metadata_ if metadata: + irm = InstalledRepositoryManager(self.app) # Handle repository dependencies. ( installed_repository_dependencies, missing_repository_dependencies, - ) = self.app.installed_repository_manager.get_installed_and_missing_repository_dependencies(repository) + ) = irm.get_installed_and_missing_repository_dependencies(repository) # Handle the current repository's tool dependencies. repository_tool_dependencies = metadata.get("tool_dependencies", None) # Make sure to display missing tool dependencies as well. @@ -290,3 +292,20 @@ def build_repository_containers( except Exception as e: log.debug(f"Exception in build_repository_containers: {str(e)}") return containers_dict + + +def build_manage_repository_dict(app, status, repository): + dd = DependencyDisplayer(app) + containers_dict = dd.populate_containers_dict_from_repository_metadata( + repository=repository, + ) + management_dict = { + "status": status, + } + missing_repo_dependencies = containers_dict.get("missing_repository_dependencies", None) + if missing_repo_dependencies: + management_dict["missing_repository_dependencies"] = missing_repo_dependencies.to_dict() + repository_dependencies = containers_dict.get("repository_dependencies", None) + if repository_dependencies: + management_dict["repository_dependencies"] = repository_dependencies.to_dict() + return management_dict diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 859d5743f5da..544f1a9a1ef1 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -153,17 +153,4 @@ def _manage_repository_json(self, trans, **kwd): trans.install_model.context.add(repository) trans.install_model.context.flush() message = "The repository information has been updated." - dd = dependency_display.DependencyDisplayer(trans.app) - containers_dict = dd.populate_containers_dict_from_repository_metadata( - repository=repository, - ) - management_dict = { - "status": status, - } - missing_repo_dependencies = containers_dict.get("missing_repository_dependencies", None) - if missing_repo_dependencies: - management_dict["missing_repository_dependencies"] = missing_repo_dependencies.to_dict() - repository_dependencies = containers_dict.get("repository_dependencies", None) - if repository_dependencies: - management_dict["repository_dependencies"] = repository_dependencies.to_dict() - return management_dict + return dependency_display.build_manage_repository_dict(trans.app, status, repository) diff --git a/lib/tool_shed/test/base/test_db_util.py b/lib/tool_shed/test/base/test_db_util.py index 76fcd7792664..3a5892e3ea0e 100644 --- a/lib/tool_shed/test/base/test_db_util.py +++ b/lib/tool_shed/test/base/test_db_util.py @@ -1,5 +1,8 @@ import logging -from typing import Optional +from typing import ( + List, + Optional, +) from sqlalchemy import ( and_, @@ -38,22 +41,19 @@ def get_all_repositories(): return sa_session.query(model.Repository).all() -def get_all_installed_repositories(actually_installed=False): - if actually_installed: - return ( - install_session.query(galaxy.model.tool_shed_install.ToolShedRepository) - .filter( - and_( - galaxy.model.tool_shed_install.ToolShedRepository.table.c.deleted == false(), - galaxy.model.tool_shed_install.ToolShedRepository.table.c.uninstalled == false(), - galaxy.model.tool_shed_install.ToolShedRepository.table.c.status - == galaxy.model.tool_shed_install.ToolShedRepository.installation_status.INSTALLED, - ) +def get_all_installed_repositories(session=install_session) -> List[galaxy.model.tool_shed_install.ToolShedRepository]: + return list( + session.query(galaxy.model.tool_shed_install.ToolShedRepository) + .filter( + and_( + galaxy.model.tool_shed_install.ToolShedRepository.table.c.deleted == false(), + galaxy.model.tool_shed_install.ToolShedRepository.table.c.uninstalled == false(), + galaxy.model.tool_shed_install.ToolShedRepository.table.c.status + == galaxy.model.tool_shed_install.ToolShedRepository.installation_status.INSTALLED, ) - .all() ) - else: - return install_session.query(galaxy.model.tool_shed_install.ToolShedRepository).all() + .all() + ) def get_galaxy_repository_by_name_owner_changeset_revision(repository_name, owner, changeset_revision): @@ -78,8 +78,8 @@ def get_installed_repository_by_id(repository_id): ) -def get_installed_repository_by_name_owner(repository_name, owner, return_multiple=False): - query = install_session.query(galaxy.model.tool_shed_install.ToolShedRepository).filter( +def get_installed_repository_by_name_owner(repository_name, owner, return_multiple=False, session=install_session): + query = session.query(galaxy.model.tool_shed_install.ToolShedRepository).filter( and_( galaxy.model.tool_shed_install.ToolShedRepository.table.c.name == repository_name, galaxy.model.tool_shed_install.ToolShedRepository.table.c.owner == owner, diff --git a/lib/tool_shed/test/base/twilltestcase.py b/lib/tool_shed/test/base/twilltestcase.py index 207f8bd0ba2b..243fb19972fc 100644 --- a/lib/tool_shed/test/base/twilltestcase.py +++ b/lib/tool_shed/test/base/twilltestcase.py @@ -4,6 +4,7 @@ import tempfile import time from json import loads +from pathlib import Path from typing import ( Any, Dict, @@ -23,9 +24,25 @@ hg, ui, ) +from sqlalchemy import ( + and_, + false, +) import galaxy.model.tool_shed_install as galaxy_model +from galaxy.schema.schema import CheckForUpdatesResponse from galaxy.security import idencoding +from galaxy.tool_shed.galaxy_install.install_manager import InstallRepositoryManager +from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager +from galaxy.tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import ( + InstalledRepositoryMetadataManager, +) +from galaxy.tool_shed.unittest_utils import ( + StandaloneInstallationTarget, + ToolShedTarget, +) +from galaxy.tool_shed.util.dependency_display import build_manage_repository_dict +from galaxy.tool_shed.util.repository_util import check_for_updates from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, smart_str, @@ -94,7 +111,7 @@ def install_repository( def reactivate_repository(self, installed_repository: galaxy_model.ToolShedRepository) -> None: ... - def reset_metadata_on_selected_installed_repositories(self, repository_ids: List[str]) -> None: + def reset_metadata_on_installed_repositories(self, repositories: List[galaxy_model.ToolShedRepository]) -> None: ... def reset_installed_repository_metadata(self, repository: galaxy_model.ToolShedRepository) -> None: @@ -108,6 +125,14 @@ def update_installed_repository( ) -> Dict[str, Any]: ... + @property + def tool_data_path(self) -> str: + ... + + @property + def shed_tool_data_table_conf(self) -> str: + ... + def get_tool_names(self) -> List[str]: ... @@ -126,6 +151,12 @@ def get_installed_repository_for( ) -> Optional[Dict[str, Any]]: ... + def get_all_installed_repositories(self) -> List[galaxy_model.ToolShedRepository]: + ... + + def refresh_tool_shed_repository(self, repo: galaxy_model.ToolShedRepository) -> None: + ... + class GalaxyInteractorToolShedInstallationClient(ToolShedInstallationClient): """A Galaxy API + Database as a installation target for the tool shed.""" @@ -148,7 +179,7 @@ def check_galaxy_repository_tool_panel_section( galaxy_model.ToolShedRepository.installation_status.UNINSTALLED, galaxy_model.ToolShedRepository.installation_status.DEACTIVATED, ]: - tool_panel_section = self._get_tool_panel_section_from_repository_metadata(metadata) + tool_panel_section = _get_tool_panel_section_from_repository_metadata(metadata) else: tool_panel_section = self._get_tool_panel_section_from_api(metadata) assert ( @@ -229,7 +260,10 @@ def reactivate_repository(self, installed_repository: galaxy_model.ToolShedRepos url = "/admin_toolshed/restore_repository" self._visit_galaxy_url(url, params=params) - def reset_metadata_on_selected_installed_repositories(self, repository_ids: List[str]) -> None: + def reset_metadata_on_installed_repositories(self, repositories: List[galaxy_model.ToolShedRepository]) -> None: + repository_ids = [] + for repository in repositories: + repository_ids.append(self.testcase.security.encode_id(repository.id)) api_key = get_admin_api_key() response = requests.post( f"{self.testcase.galaxy_url}/api/tool_shed_repositories/reset_metadata_on_selected_installed_repositories", @@ -279,6 +313,14 @@ def reset_installed_repository_metadata(self, repository: galaxy_model.ToolShedR ) assert response.status_code != 403, response.content + @property + def tool_data_path(self): + return os.environ.get("GALAXY_TEST_TOOL_DATA_PATH") + + @property + def shed_tool_data_table_conf(self): + return os.environ.get("TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF") + def get_tool_names(self) -> List[str]: response = self.testcase.galaxy_interactor._get("tools?in_panel=false") response.raise_for_status() @@ -302,6 +344,15 @@ def get_installed_repository_for( ) -> Optional[Dict[str, Any]]: return self.testcase.get_installed_repository_for(owner=owner, name=name, changeset=changeset) + def get_all_installed_repositories(self) -> List[galaxy_model.ToolShedRepository]: + repositories = test_db_util.get_all_installed_repositories() + for repository in repositories: + test_db_util.ga_refresh(repository) + return repositories + + def refresh_tool_shed_repository(self, repo: galaxy_model.ToolShedRepository) -> None: + test_db_util.ga_refresh(repo) + def _galaxy_login(self, email="test@bx.psu.edu", password="testuser", username="admin-user"): self._galaxy_logout() self._create_user_in_galaxy(email=email, password=password, username=username) @@ -340,24 +391,7 @@ def _get_tool_panel_section_from_api(self, metadata): tool_panel_section = tool_dict["panel_section_name"] return tool_panel_section - def _get_tool_panel_section_from_repository_metadata(self, metadata): - tool_metadata = metadata["tools"] - tool_guid = tool_metadata[0]["guid"] - assert "tool_panel_section" in metadata, f"Tool panel section not found in metadata: {metadata}" - tool_panel_section_metadata = metadata["tool_panel_section"] - # tool_section_dict = dict( tool_config=guids_and_configs[ guid ], - # id=section_id, - # name=section_name, - # version=section_version ) - # This dict is appended to tool_panel_section_metadata[ tool_guid ] - tool_panel_section = tool_panel_section_metadata[tool_guid][0]["name"] - return tool_panel_section - def _wait_for_repository_installation(self, repository_ids): - final_states = [ - galaxy_model.ToolShedRepository.installation_status.ERROR, - galaxy_model.ToolShedRepository.installation_status.INSTALLED, - ] # Wait until all repositories are in a final state before returning. This ensures that subsequent tests # are running against an installed repository, and not one that is still in the process of installing. if repository_ids: @@ -365,18 +399,7 @@ def _wait_for_repository_installation(self, repository_ids): galaxy_repository = test_db_util.get_installed_repository_by_id( self.testcase.security.decode_id(repository_id) ) - timeout_counter = 0 - while galaxy_repository.status not in final_states: - test_db_util.ga_refresh(galaxy_repository) - timeout_counter = timeout_counter + 1 - # This timeout currently defaults to 10 minutes. - if timeout_counter > repository_installation_timeout: - raise AssertionError( - "Repository installation timed out, %d seconds elapsed, repository state is %s." - % (timeout_counter, galaxy_repository.status) - ) - break - time.sleep(1) + _wait_for_installation(galaxy_repository, test_db_util.ga_refresh) def _visit_galaxy_url(self, url, params=None, doseq=False, allowed_codes=None): if allowed_codes is None: @@ -385,10 +408,186 @@ def _visit_galaxy_url(self, url, params=None, doseq=False, allowed_codes=None): self.testcase.visit_url(url, params=params, doseq=doseq, allowed_codes=allowed_codes) +class StandaloneToolShedInstallationClient(ToolShedInstallationClient): + def __init__(self, testcase): + self.testcase = testcase + self.temp_directory = Path(tempfile.mkdtemp(prefix="toolshedtestinstalltarget")) + tool_shed_target = ToolShedTarget( + self.testcase.url, + "Tool Shed for Testing", + ) + self._installation_target = StandaloneInstallationTarget(self.temp_directory, tool_shed_target=tool_shed_target) + + def setup(self) -> None: + pass + + def check_galaxy_repository_tool_panel_section( + self, repository: galaxy_model.ToolShedRepository, expected_tool_panel_section: str + ) -> None: + metadata = repository.metadata_ + assert "tools" in metadata, f"Tools not found in repository metadata: {metadata}" + # TODO: check actual toolbox if tool is already installed... + tool_panel_section = _get_tool_panel_section_from_repository_metadata(metadata) + assert ( + tool_panel_section == expected_tool_panel_section + ), f"Expected to find tool panel section *{expected_tool_panel_section}*, but instead found *{tool_panel_section}*\nMetadata: {metadata}\n" + + def deactivate_repository(self, installed_repository: galaxy_model.ToolShedRepository) -> None: + irm = InstalledRepositoryManager(app=self._installation_target) + errors = irm.uninstall_repository(repository=installed_repository, remove_from_disk=False) + if errors: + raise Exception( + f"Attempting to uninstall tool dependencies for repository named {installed_repository.name} resulted in errors: {errors}" + ) + + def display_installed_jobs_list_page( + self, installed_repository: galaxy_model.ToolShedRepository, data_manager_names=None, strings_displayed=None + ) -> None: + raise NotImplementedError() + + def installed_repository_extended_info( + self, installed_repository: galaxy_model.ToolShedRepository + ) -> Dict[str, Any]: + self._installation_target.install_model.context.refresh(installed_repository) + return build_manage_repository_dict(self._installation_target, "ok", installed_repository) + + def install_repository( + self, + name: str, + owner: str, + changeset_revision: str, + install_tool_dependencies: bool, + install_repository_dependencies: bool, + new_tool_panel_section_label: Optional[str], + ): + tool_shed_url = self.testcase.url + payload = { + "tool_shed_url": tool_shed_url, + "name": name, + "owner": owner, + "changeset_revision": changeset_revision, + "install_tool_dependencies": install_tool_dependencies, + "install_repository_dependencies": install_repository_dependencies, + "install_resolver_dependencies": False, + } + if new_tool_panel_section_label: + payload["new_tool_panel_section_label"] = new_tool_panel_section_label + irm = InstallRepositoryManager(app=self._installation_target) + installed_tool_shed_repositories = irm.install(str(tool_shed_url), name, owner, changeset_revision, payload) + for installed_tool_shed_repository in installed_tool_shed_repositories or []: + _wait_for_installation( + installed_tool_shed_repository, self._installation_target.install_model.context.refresh + ) + + def reactivate_repository(self, installed_repository: galaxy_model.ToolShedRepository) -> None: + irm = InstalledRepositoryManager(app=self._installation_target) + irm.activate_repository(installed_repository) + + def reset_metadata_on_installed_repositories(self, repositories: List[galaxy_model.ToolShedRepository]) -> None: + for repository in repositories: + irmm = InstalledRepositoryMetadataManager(self._installation_target) + irmm.set_repository(repository) + irmm.reset_all_metadata_on_installed_repository() + + def reset_installed_repository_metadata(self, repository: galaxy_model.ToolShedRepository) -> None: + irmm = InstalledRepositoryMetadataManager(self._installation_target) + irmm.set_repository(repository) + irmm.reset_all_metadata_on_installed_repository() + + def uninstall_repository(self, installed_repository: galaxy_model.ToolShedRepository) -> None: + irm = InstalledRepositoryManager(app=self._installation_target) + errors = irm.uninstall_repository(repository=installed_repository, remove_from_disk=True) + if errors: + raise Exception( + f"Attempting to uninstall tool dependencies for repository named {installed_repository.name} resulted in errors: {errors}" + ) + + def update_installed_repository( + self, installed_repository: galaxy_model.ToolShedRepository, verify_no_updates: bool = False + ) -> Dict[str, Any]: + message, status = check_for_updates( + self._installation_target.tool_shed_registry, + self._installation_target.install_model.context, + installed_repository.id, + ) + response = CheckForUpdatesResponse(message=message, status=status) + response_dict = response.dict() + if verify_no_updates: + assert "message" in response_dict + message = response_dict["message"] + assert "The status has not changed in the tool shed for repository" in message, str(response_dict) + return response_dict + + def get_installed_repository_by_name_owner( + self, repository_name: str, repository_owner: str + ) -> galaxy_model.ToolShedRepository: + return test_db_util.get_installed_repository_by_name_owner( + repository_name, repository_owner, session=self._installation_target.install_model.context + ) + + def get_installed_repositories_by_name_owner( + self, repository_name: str, repository_owner: str + ) -> List[galaxy_model.ToolShedRepository]: + return test_db_util.get_installed_repository_by_name_owner( + repository_name, + repository_owner, + return_multiple=True, + session=self._installation_target.install_model.context, + ) + + def get_installed_repository_for( + self, owner: Optional[str] = None, name: Optional[str] = None, changeset: Optional[str] = None + ) -> Optional[Dict[str, Any]]: + clause_list = [] + if name is not None: + clause_list.append(galaxy_model.ToolShedRepository.table.c.name == name) + if owner is not None: + clause_list.append(galaxy_model.ToolShedRepository.table.c.owner == owner) + if changeset is not None: + clause_list.append(galaxy_model.ToolShedRepository.table.c.changeset_revision == changeset) + clause_list.append(galaxy_model.ToolShedRepository.table.c.deleted == false()) + clause_list.append(galaxy_model.ToolShedRepository.table.c.uninstalled == false()) + + query = self._installation_target.install_model.context.query(galaxy_model.ToolShedRepository) + if len(clause_list) > 0: + query = query.filter(and_(*clause_list)) + repository = query.one_or_none() + if repository: + return repository.to_dict() + else: + return None + + def get_all_installed_repositories(self) -> List[galaxy_model.ToolShedRepository]: + repositories = test_db_util.get_all_installed_repositories( + session=self._installation_target.install_model.context + ) + for repository in repositories: + self._installation_target.install_model.context.refresh(repository) + return repositories + + def refresh_tool_shed_repository(self, repo: galaxy_model.ToolShedRepository) -> None: + self._installation_target.install_model.context.refresh(repo) + + @property + def shed_tool_data_table_conf(self): + return self._installation_target.config.shed_tool_data_table_config + + @property + def tool_data_path(self): + return self._installation_target.config.tool_data_path + + def get_tool_names(self) -> List[str]: + tool_names = [] + for _, tool in self._installation_target.toolbox.tools(): + tool_names.append(tool.name) + return tool_names + + class ShedTwillTestCase(ShedApiTestCase): """Class of FunctionalTestCase geared toward HTML interactions using the Twill library.""" requires_galaxy: bool = False + _installation_client = None def setUp(self): super().setUp() @@ -399,14 +598,22 @@ def setUp(self): self.hgweb_config_manager = hgweb_config.hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.hgweb_config_dir self.tool_shed_test_tmp_dir = os.environ.get("TOOL_SHED_TEST_TMP_DIR", None) - self.shed_tool_data_table_conf = os.environ.get("TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF") self.file_dir = os.environ.get("TOOL_SHED_TEST_FILE_DIR", None) - self.tool_data_path = os.environ.get("GALAXY_TEST_TOOL_DATA_PATH") self.shed_tool_conf = os.environ.get("GALAXY_TEST_SHED_TOOL_CONF") self.test_db_util = test_db_util - self._installation_client = GalaxyInteractorToolShedInstallationClient(self) - if self.requires_galaxy: - self._installation_client.setup() + if os.environ.get("TOOL_SHED_TEST_INSTALL_CLIENT") == "standalone": + # TODO: once nose is out of the way - try to get away without + # instantiating the unused Galaxy server here. + installation_client_class = StandaloneToolShedInstallationClient + full_stack_galaxy = False + else: + installation_client_class = GalaxyInteractorToolShedInstallationClient + full_stack_galaxy = True + self.full_stack_galaxy = full_stack_galaxy + if self.requires_galaxy and (self.__class__._installation_client is None): + self.__class__._installation_client = installation_client_class(self) + self.__class__._installation_client.setup() + self._installation_client = self.__class__._installation_client def check_for_strings(self, strings_displayed=None, strings_not_displayed=None): strings_displayed = strings_displayed or [] @@ -638,6 +845,7 @@ def check_for_valid_tools(self, repository, strings_displayed=None, strings_not_ def check_galaxy_repository_db_status(self, repository_name, owner, expected_status): installed_repository = self._get_installed_repository_by_name_owner(repository_name, owner) + self._refresh_tool_shed_repository(installed_repository) assert ( installed_repository.status == expected_status ), f"Status in database is {installed_repository.status}, expected {expected_status}" @@ -747,6 +955,7 @@ def check_string_count_in_page(self, pattern, min_count, max_count=None): def check_galaxy_repository_tool_panel_section( self, repository: galaxy_model.ToolShedRepository, expected_tool_panel_section: str ) -> None: + assert self._installation_client self._installation_client.check_galaxy_repository_tool_panel_section(repository, expected_tool_panel_section) def clone_repository(self, repository: Repository, destination_path: str) -> None: @@ -828,6 +1037,7 @@ def create_repository_dependency( ) def deactivate_repository(self, installed_repository: galaxy_model.ToolShedRepository) -> None: + assert self._installation_client self._installation_client.deactivate_repository(installed_repository) def delete_files_from_repository( @@ -858,11 +1068,13 @@ def delete_repository(self, repository: Repository) -> None: self.check_for_strings(strings_displayed, strings_not_displayed) def display_installed_jobs_list_page(self, installed_repository, data_manager_names=None, strings_displayed=None): + assert self._installation_client self._installation_client.display_installed_jobs_list_page( installed_repository, data_manager_names, strings_displayed ) def display_installed_repository_manage_json(self, installed_repository): + assert self._installation_client return self._installation_client.installed_repository_extended_info(installed_repository) def display_manage_repository_page( @@ -1050,6 +1262,10 @@ def generate_temp_path(self, test_script_path, additional_paths=None): os.makedirs(temp_path) return temp_path + def get_all_installed_repositories(self) -> List[galaxy_model.ToolShedRepository]: + assert self._installation_client + return self._installation_client.get_all_installed_repositories() + def get_filename(self, filename, filepath=None): if filepath is not None: return os.path.abspath(os.path.join(filepath, filename)) @@ -1259,6 +1475,7 @@ def _install_repository( # repository_id = repository.id if changeset_revision is None: changeset_revision = self.get_repository_tip(repository) + assert self._installation_client self._installation_client.install_repository( name, owner, @@ -1398,8 +1615,9 @@ def reset_metadata_on_selected_repositories(self, repository_ids): kwd = dict(repository_ids=repository_ids) self.submit_form(button="reset_metadata_on_selected_repositories_button", **kwd) - def reset_metadata_on_selected_installed_repositories(self, repository_ids): - self._installation_client.reset_metadata_on_selected_installed_repositories(repository_ids) + def reset_metadata_on_installed_repositories(self, repositories): + assert self._installation_client + self._installation_client.reset_metadata_on_installed_repositories(repositories) def reset_repository_metadata(self, repository): params = {"id": repository.id} @@ -1490,11 +1708,13 @@ def undelete_repository(self, repository: Repository) -> None: self.check_for_strings(strings_displayed, strings_not_displayed) def _uninstall_repository(self, installed_repository: galaxy_model.ToolShedRepository) -> None: + assert self._installation_client self._installation_client.uninstall_repository(installed_repository) def update_installed_repository( self, installed_repository: galaxy_model.ToolShedRepository, verify_no_updates: bool = False ) -> Dict[str, Any]: + assert self._installation_client return self._installation_client.update_installed_repository(installed_repository, verify_no_updates=False) def upload_file( @@ -1600,7 +1820,9 @@ def verify_installed_repositories(self, installed_repositories=None, uninstalled def verify_installed_repository_metadata_unchanged(self, name, owner): installed_repository = self._get_installed_repository_by_name_owner(name, owner) + assert installed_repository metadata = installed_repository.metadata_ + assert self._installation_client self._installation_client.reset_installed_repository_metadata(installed_repository) new_metadata = installed_repository.metadata_ assert metadata == new_metadata, f"Metadata for installed repository {name} differs after metadata reset." @@ -1610,9 +1832,27 @@ def verify_installed_repository_no_tool_panel_section(self, repository): metadata = repository.metadata_ assert "tool_panel_section" not in metadata, f"Tool panel section incorrectly found in metadata: {metadata}" + @property + def shed_tool_data_table_conf(self): + return self._installation_client.shed_tool_data_table_conf + + @property + def tool_data_path(self): + return self._installation_client.tool_data_path + + def _refresh_tool_shed_repository(self, repo: galaxy_model.ToolShedRepository) -> None: + assert self._installation_client + self._installation_client.refresh_tool_shed_repository(repo) + def verify_installed_repository_data_table_entries(self, required_data_table_entries): # The value of the received required_data_table_entries will be something like: [ 'sam_fa_indexes' ] - data_tables, error_message = xml_util.parse_xml(self.shed_tool_data_table_conf) + shed_tool_data_table_conf = self.shed_tool_data_table_conf + data_tables, error_message = xml_util.parse_xml(shed_tool_data_table_conf) + with open(shed_tool_data_table_conf) as f: + shed_tool_data_table_conf_contents = f.read() + assert ( + not error_message + ), f"Failed to parse {shed_tool_data_table_conf} properly. File contents [{shed_tool_data_table_conf_contents}]" found = False # With the tool shed, the "path" attribute that is hard-coded into the tool_data_tble_conf.xml # file is ignored. This is because the tool shed requires the directory location to which this @@ -1668,21 +1908,32 @@ def verify_installed_repository_data_table_entries(self, required_data_table_ent break # We better have an entry like: in our parsed data_tables # or we know that the repository was not correctly installed! - assert found, f"No entry for {required_data_table_entry} in {self.shed_tool_data_table_conf}." + if not found: + if required_data_table_entry is None: + raise AssertionError( + f"No tables found in {shed_tool_data_table_conf}. File contents {shed_tool_data_table_conf_contents}" + ) + else: + raise AssertionError( + f"No entry for {required_data_table_entry} in {shed_tool_data_table_conf}. File contents {shed_tool_data_table_conf_contents}" + ) def _get_installed_repository_by_name_owner( self, repository_name: str, repository_owner: str ) -> galaxy_model.ToolShedRepository: + assert self._installation_client return self._installation_client.get_installed_repository_by_name_owner(repository_name, repository_owner) def _get_installed_repositories_by_name_owner( self, repository_name: str, repository_owner: str ) -> List[galaxy_model.ToolShedRepository]: + assert self._installation_client return self._installation_client.get_installed_repositories_by_name_owner(repository_name, repository_owner) def _get_installed_repository_for( self, owner: Optional[str] = None, name: Optional[str] = None, changeset: Optional[str] = None ): + assert self._installation_client return self._installation_client.get_installed_repository_for(owner=owner, name=name, changeset=changeset) def _assert_has_installed_repos_with_names(self, *names): @@ -1721,8 +1972,9 @@ def _assert_has_installed_repository_dependency( changeset: Optional[str] = None, ) -> None: json = self.display_installed_repository_manage_json(installed_repository) - assert "repository_dependencies" in json, ( - "No repository dependencies were defined in %s." % installed_repository.name + assert "repository_dependencies" in json, "No repository dependencies were defined in %s. manage json is %s" % ( + installed_repository.name, + json, ) repository_dependencies = json["repository_dependencies"] found = False @@ -1755,6 +2007,7 @@ def _assert_is_not_missing_dependency( def _assert_has_valid_tool_with_name(self, tool_name: str) -> None: def assert_has(): + assert self._installation_client tool_names = self._installation_client.get_tool_names() assert tool_name in tool_names @@ -1798,3 +2051,33 @@ def verify_unchanged_repository_metadata(self, repository: Repository): # Python's dict comparison recursively compares sorted key => value pairs and returns true if any key or value differs, # or if the number of keys differs. assert old_metadata == new_metadata, f"Metadata changed after reset on repository {repository.name}." + + +def _wait_for_installation(repository: galaxy_model.ToolShedRepository, refresh): + final_states = [ + galaxy_model.ToolShedRepository.installation_status.ERROR, + galaxy_model.ToolShedRepository.installation_status.INSTALLED, + ] + # Wait until all repositories are in a final state before returning. This ensures that subsequent tests + # are running against an installed repository, and not one that is still in the process of installing. + timeout_counter = 0 + while repository.status not in final_states: + refresh(repository) + timeout_counter = timeout_counter + 1 + # This timeout currently defaults to 10 minutes. + if timeout_counter > repository_installation_timeout: + raise AssertionError( + "Repository installation timed out, %d seconds elapsed, repository state is %s." + % (timeout_counter, repository.status) + ) + break + time.sleep(1) + + +def _get_tool_panel_section_from_repository_metadata(metadata): + tool_metadata = metadata["tools"] + tool_guid = tool_metadata[0]["guid"] + assert "tool_panel_section" in metadata, f"Tool panel section not found in metadata: {metadata}" + tool_panel_section_metadata = metadata["tool_panel_section"] + tool_panel_section = tool_panel_section_metadata[tool_guid][0]["name"] + return tool_panel_section diff --git a/lib/tool_shed/test/functional/test_1040_install_repository_basic_circular_dependencies.py b/lib/tool_shed/test/functional/test_1040_install_repository_basic_circular_dependencies.py index 5dfdcf31fd6c..c71ad0c31ca0 100644 --- a/lib/tool_shed/test/functional/test_1040_install_repository_basic_circular_dependencies.py +++ b/lib/tool_shed/test/functional/test_1040_install_repository_basic_circular_dependencies.py @@ -147,7 +147,7 @@ def test_0030_uninstall_filtering_repository(self): common.test_user_1, filtering_repository_name, installed_filtering_repository.installed_changeset_revision ) self.deactivate_repository(installed_filtering_repository) - self.test_db_util.ga_refresh(installed_filtering_repository) + self._refresh_tool_shed_repository(installed_filtering_repository) self._assert_has_missing_dependency(installed_freebayes_repository, filtering_repository_name) self.check_galaxy_repository_db_status(filtering_repository_name, common.test_user_1_name, "Deactivated") @@ -185,7 +185,7 @@ def test_0040_uninstall_freebayes_repository(self): assert not self._get_installed_repository_for( common.test_user_1, freebayes_repository_name, installed_freebayes_repository.installed_changeset_revision ) - self.test_db_util.ga_refresh(installed_freebayes_repository) + self._refresh_tool_shed_repository(installed_filtering_repository) self._assert_has_missing_dependency(installed_filtering_repository, freebayes_repository_name) self.check_galaxy_repository_db_status("freebayes_0040", "user1", "Deactivated") @@ -207,6 +207,6 @@ def test_0045_deactivate_filtering_repository(self): assert not self._get_installed_repository_for( common.test_user_1, filtering_repository_name, installed_filtering_repository.installed_changeset_revision ) - self.test_db_util.ga_refresh(installed_filtering_repository) + self._refresh_tool_shed_repository(installed_freebayes_repository) self._assert_has_missing_dependency(installed_freebayes_repository, filtering_repository_name) self.check_galaxy_repository_db_status(filtering_repository_name, common.test_user_1_name, "Deactivated") diff --git a/lib/tool_shed/test/functional/test_1050_circular_dependencies_4_levels.py b/lib/tool_shed/test/functional/test_1050_circular_dependencies_4_levels.py index 90fb8db3e309..713e7f7e84be 100644 --- a/lib/tool_shed/test/functional/test_1050_circular_dependencies_4_levels.py +++ b/lib/tool_shed/test/functional/test_1050_circular_dependencies_4_levels.py @@ -389,7 +389,6 @@ def test_0065_deactivate_bismark_repository(self): (column_repository_name, common.test_user_1_name), (emboss_repository_name, common.test_user_1_name), (convert_repository_name, common.test_user_1_name), - (bismark_repository_name, common.test_user_1_name), ] strings_displayed = ["emboss_0050", "column_maker_0050", "convert_chars_0050"] strings_not_displayed = ["bismark", "filtering_0050", "freebayes_0050"] @@ -402,7 +401,7 @@ def test_0070_uninstall_emboss_repository(self): repository = self._get_installed_repository_by_name_owner(emboss_repository_name, common.test_user_1_name) self._uninstall_repository(repository) self._assert_has_no_installed_repos_with_names(repository.name) - self.test_db_util.ga_refresh(repository) + self._refresh_tool_shed_repository(repository) self.check_galaxy_repository_tool_panel_section(repository, "emboss_5_0050") # Now we have bismark, column_maker, and convert_chars installed, filtering and freebayes never installed, # and emboss uninstalled. diff --git a/lib/tool_shed/test/functional/test_1080_advanced_circular_dependency_installation.py b/lib/tool_shed/test/functional/test_1080_advanced_circular_dependency_installation.py index dd31fb94f8c6..f9352bcb05f3 100644 --- a/lib/tool_shed/test/functional/test_1080_advanced_circular_dependency_installation.py +++ b/lib/tool_shed/test/functional/test_1080_advanced_circular_dependency_installation.py @@ -135,15 +135,18 @@ def test_0020_install_convert_repository(self): column_repository_name, common.test_user_1_name ) assert self._get_installed_repository_for( - common.test_user_1, "convert_chars_0080", installed_convert_repository.installed_changeset_revision - ) - self._assert_has_installed_repository_dependency( - installed_convert_repository, "column_maker_0080", installed_column_repository.installed_changeset_revision - ) - # installed_convert_repository has required_repositories and the following string - # is included when not installing via the API. This distrubs me but we've not installed - # not from the API for a long time so I'm just dropping the check. -John - # "Missing repository dependencies", + common.test_user_1, convert_repository_name, installed_convert_repository.installed_changeset_revision + ) + if self.full_stack_galaxy: + # This branch has been broken since we switched from mako to API for installing... + self._assert_has_installed_repository_dependency( + installed_convert_repository, + column_repository_name, + installed_column_repository.installed_changeset_revision, + ) + else: + # Previous mako had some string checks and such equivalent to this. + self._assert_has_missing_dependency(installed_convert_repository, column_repository_name) def test_0025_install_column_repository(self): """Install column maker with repository dependencies into column_maker tool panel section.""" diff --git a/lib/tool_shed/test/functional/test_1090_repository_dependency_handling.py b/lib/tool_shed/test/functional/test_1090_repository_dependency_handling.py index 02a80ad62895..0a249455275c 100644 --- a/lib/tool_shed/test/functional/test_1090_repository_dependency_handling.py +++ b/lib/tool_shed/test/functional/test_1090_repository_dependency_handling.py @@ -115,6 +115,7 @@ def test_0020_install_repositories(self): installed_convert_repository = self._get_installed_repository_by_name_owner( convert_repository_name, common.test_user_1_name ) + self._refresh_tool_shed_repository(installed_convert_repository) self._assert_has_installed_repos_with_names("convert_chars_1085", "column_maker_1085") self._assert_is_not_missing_dependency(installed_convert_repository, "column_maker_1085") @@ -124,7 +125,6 @@ def test_0025_uninstall_column_repository(self): column_repository_name, common.test_user_1_name ) self._uninstall_repository(installed_column_repository) - self.test_db_util.ga_refresh(installed_column_repository) self.check_galaxy_repository_tool_panel_section(installed_column_repository, "column_maker") def test_0030_uninstall_convert_repository(self): @@ -132,7 +132,7 @@ def test_0030_uninstall_convert_repository(self): convert_repository_name, common.test_user_1_name ) self._uninstall_repository(installed_convert_repository) - self.test_db_util.ga_refresh(installed_convert_repository) + self._refresh_tool_shed_repository(installed_convert_repository) self.check_galaxy_repository_tool_panel_section(installed_convert_repository, "column_maker") def test_0035_reinstall_column_repository(self): diff --git a/lib/tool_shed/test/functional/test_1120_install_repository_with_complex_dependencies.py b/lib/tool_shed/test/functional/test_1120_install_repository_with_complex_dependencies.py index 8e41e0c5b20b..b3fb64a7d6d7 100644 --- a/lib/tool_shed/test/functional/test_1120_install_repository_with_complex_dependencies.py +++ b/lib/tool_shed/test/functional/test_1120_install_repository_with_complex_dependencies.py @@ -263,7 +263,6 @@ def test_0045_install_base_repository(self): bwa_base_repository_name, common.test_user_1_name, category_name, - install_tool_dependencies=True, preview_strings_displayed=preview_strings_displayed, ) diff --git a/lib/tool_shed/test/functional/test_1180_circular_prior_installation_required.py b/lib/tool_shed/test/functional/test_1180_circular_prior_installation_required.py index 91df96ba0658..92d8dd89bdcd 100644 --- a/lib/tool_shed/test/functional/test_1180_circular_prior_installation_required.py +++ b/lib/tool_shed/test/functional/test_1180_circular_prior_installation_required.py @@ -283,8 +283,9 @@ def test_0050_verify_reinstallation_order(self): convert_repository_name, common.test_user_1_name ) # Filtering was selected for reinstallation, so convert chars and column maker should have been installed first. - for repo in [convert_repository, column_repository, filter_repository]: - self.test_db_util.install_session.refresh(repo) + if self.full_stack_galaxy: + for repo in [convert_repository, column_repository, filter_repository]: + self.test_db_util.install_session.refresh(repo) assert ( filter_repository.update_time > convert_repository.update_time ), "Prior installed convert_chars_0160 shows a later update time than filtering_0160" diff --git a/lib/tool_shed/test/functional/test_1300_reset_all_metadata.py b/lib/tool_shed/test/functional/test_1300_reset_all_metadata.py index 26f7db2239fd..eda3d6dc693b 100644 --- a/lib/tool_shed/test/functional/test_1300_reset_all_metadata.py +++ b/lib/tool_shed/test/functional/test_1300_reset_all_metadata.py @@ -526,14 +526,13 @@ def test_9900_install_all_missing_repositories(self): def test_9905_reset_metadata_on_all_repositories(self): """Reset metadata on all repositories, then verify that it has not changed.""" - repository_metadata = dict() - repositories = self.test_db_util.get_all_installed_repositories(actually_installed=True) + repositories = self.get_all_installed_repositories() + repository_metadata = {} for repository in repositories: - repository_metadata[self.security.encode_id(repository.id)] = repository.metadata_ - self.reset_metadata_on_selected_installed_repositories(list(repository_metadata.keys())) - for repository in repositories: - self.test_db_util.ga_refresh(repository) - old_metadata = repository_metadata[self.security.encode_id(repository.id)] + repository_metadata[repository.id] = repository.metadata_ + self.reset_metadata_on_installed_repositories(repositories) + for repository in self.get_all_installed_repositories(): + old_metadata = repository_metadata[repository.id] # When a repository with tools to be displayed in a tool panel section is deactivated and reinstalled, # the tool panel section remains in the repository metadata. However, when the repository's metadata # is subsequently reset, the tool panel section is removed from the repository metadata. While this diff --git a/lib/tool_shed/test/functional/test_1460_data_managers.py b/lib/tool_shed/test/functional/test_1460_data_managers.py index e26853817aab..7269e72fd12d 100644 --- a/lib/tool_shed/test/functional/test_1460_data_managers.py +++ b/lib/tool_shed/test/functional/test_1460_data_managers.py @@ -77,16 +77,17 @@ def test_0020_install_data_manager_repository(self): data_manager_repository_name, common.test_user_1_name, category_name, - install_tool_dependencies=True, + install_tool_dependencies=False, ) def test_0030_verify_data_manager_tool(self): """Verify that the data_manager_1460 repository is installed and Data Manager tool appears in list in Galaxy.""" repository = self._get_installed_repository_by_name_owner(data_manager_repository_name, common.test_user_1_name) - strings_displayed = ["status", "jobs", data_manager_name] - self.display_installed_jobs_list_page( - repository, data_manager_names=data_manager_name, strings_displayed=strings_displayed - ) + if self.full_stack_galaxy: + strings_displayed = ["status", "jobs", data_manager_name] + self.display_installed_jobs_list_page( + repository, data_manager_names=data_manager_name, strings_displayed=strings_displayed + ) def test_0040_verify_data_manager_data_table(self): """Verify that the installed repository populated shed_tool_data_table.xml and the sample files."""