From c6bd9a7a92592a8a700b26d659cf9232bd9eaba5 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Thu, 22 Jun 2023 10:24:45 +0100 Subject: [PATCH] Address parallel install errors with ansible-galaxy on GHA (#3585) --- .github/workflows/tox.yml | 11 +---------- src/ansiblelint/__main__.py | 14 ++++++++------ src/ansiblelint/config.py | 7 +------ src/ansiblelint/runner.py | 2 +- src/ansiblelint/schemas/__store__.json | 2 +- tox.ini | 5 +++-- 6 files changed, 15 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index b1ce2f6a03..a3fe93f1ca 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -95,16 +95,7 @@ jobs: ~/.cache/ansible-compat ~/.ansible/collections ~/.ansible/roles - key: ${{ matrix.name || matrix.passed_name }}-${{ hashFiles('tools/test-eco.sh', 'requirements.yml') }} - - - name: Set galaxy cache - uses: actions/cache@v3 - if: ${{ startsWith(matrix.passed_name, 'py') }} - with: - path: | - examples/playbooks/collections/*.tar.gz - examples/playbooks/collections/ansible_collections - key: galaxy-${{ hashFiles('examples/playbooks/collections/requirements.yml') }} + key: ${{ matrix.name || matrix.passed_name }}-${{ hashFiles('tools/test-eco.sh', 'requirements.yml', 'examples/playbooks/collections/requirements.yml') }} - name: Set up Python ${{ matrix.python_version || '3.9' }} if: "!contains(matrix.shell, 'wsl')" diff --git a/src/ansiblelint/__main__.py b/src/ansiblelint/__main__.py index f6b6edfbe3..af434d046e 100755 --- a/src/ansiblelint/__main__.py +++ b/src/ansiblelint/__main__.py @@ -66,6 +66,7 @@ _logger = logging.getLogger(__name__) +cache_dir_lock: None | FileLock = None class LintLogHandler(logging.Handler): @@ -130,11 +131,12 @@ def initialize_options(arguments: list[str] | None = None) -> None: if options.cache_dir: options.cache_dir.mkdir(parents=True, exist_ok=True) - options.cache_dir_lock = None if not options.offline: # pragma: no cover - options.cache_dir_lock = FileLock(f"{options.cache_dir}/.lock") + cache_dir_lock = FileLock( # pylint: disable=redefined-outer-name + f"{options.cache_dir}/.lock", + ) try: - options.cache_dir_lock.acquire(timeout=180) + cache_dir_lock.acquire(timeout=180) except Timeout: # pragma: no cover _logger.error( "Timeout waiting for another instance of ansible-lint to release the lock.", @@ -294,9 +296,9 @@ def main(argv: list[str] | None = None) -> int: app.render_matches(result.matches) _perform_mockings_cleanup(app.options) - if options.cache_dir_lock: - options.cache_dir_lock.release() - pathlib.Path(options.cache_dir_lock.lock_file).unlink(missing_ok=True) + if cache_dir_lock: + cache_dir_lock.release() + pathlib.Path(cache_dir_lock.lock_file).unlink(missing_ok=True) if options.mock_filters: _logger.warning( "The following filters were mocked during the run: %s", diff --git a/src/ansiblelint/config.py b/src/ansiblelint/config.py index 98586a258f..6164b109df 100644 --- a/src/ansiblelint/config.py +++ b/src/ansiblelint/config.py @@ -12,7 +12,7 @@ from functools import lru_cache from importlib.metadata import PackageNotFoundError, distribution, version from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any from urllib.error import HTTPError, URLError from packaging.version import Version @@ -20,10 +20,6 @@ from ansiblelint import __version__ from ansiblelint.loaders import yaml_from_file -if TYPE_CHECKING: - from filelock import BaseFileLock - - _logger = logging.getLogger(__name__) @@ -152,7 +148,6 @@ class Options: # pylint: disable=too-many-instance-attributes,too-few-public-me config_file: str | None = None generate_ignore: bool = False rulesdir: list[Path] = field(default_factory=list) - cache_dir_lock: BaseFileLock | None = None use_default_rules: bool = False version: bool = False # display version command list_profiles: bool = False # display profiles command diff --git a/src/ansiblelint/runner.py b/src/ansiblelint/runner.py index 8fc8519265..1846d623af 100644 --- a/src/ansiblelint/runner.py +++ b/src/ansiblelint/runner.py @@ -219,7 +219,7 @@ def _run(self) -> list[MatchError]: ) # -- phase 1 : syntax check in parallel -- - app = get_app(offline=None) + app = get_app(offline=True) def worker(lintable: Lintable) -> list[MatchError]: # pylint: disable=protected-access diff --git a/src/ansiblelint/schemas/__store__.json b/src/ansiblelint/schemas/__store__.json index 8ef2fe218d..d4bcdca431 100644 --- a/src/ansiblelint/schemas/__store__.json +++ b/src/ansiblelint/schemas/__store__.json @@ -24,7 +24,7 @@ "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/inventory.json" }, "meta": { - "etag": "230d2476c1e1f60cd05cba2ac7c42ef658588b1f502e68e85404fd524ef3b3b2", + "etag": "0f376059285181985711b4271a6ff34a8dde662b9fc221d09bdcd64e4fbf86bf", "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/meta.json" }, "meta-runtime": { diff --git a/tox.ini b/tox.ini index 552fa68563..173880cf7d 100644 --- a/tox.ini +++ b/tox.ini @@ -26,9 +26,8 @@ deps = extras = test commands_pre = - # fail if submodules are not initialized - sh -c 'git submodule status --cached | grep "^ -" && { echo "Repository was not cloned recursively, please run: git submodule update --init" && exit 99; } || true' sh -c "rm -f .tox/.coverage.* 2>/dev/null || true" + bash ./tools/install-reqs.sh commands = # safety measure to assure we do not accidentally run tests with broken dependencies {envpython} -m pip check @@ -69,6 +68,7 @@ setenv = PRE_COMMIT_COLOR = always FORCE_COLOR = 1 allowlist_externals = + bash find git pwd @@ -220,6 +220,7 @@ skip_install = true usedevelop = false setenv = COVERAGE_PROCESS_START={toxinidir}/pyproject.toml +commands_pre = commands = python3 -m coverage --version # needed by codecov github actions, also ignored result to reach report one.