From 3598c43c9cdba42ab9b9be18f7cb636364528387 Mon Sep 17 00:00:00 2001 From: David Weinehall Date: Thu, 18 Apr 2024 03:23:54 +0300 Subject: [PATCH] cmtadm,*: Fix version check for Kubernetes. Again. The previous version check fix was necessary but not enough. This should *hopefully* fully fix the version check for Kubernetes. It also adds a --force option to override the cooldown period, as well as adds version checks for antctl, calicoctl, and cilium-cli. Signed-off-by: David Weinehall --- CHANGELOG/CHANGELOG-0.8.md | 396 +++++++++++++++++++++++++++++++++++++ CHANGELOG/latest.yaml | 4 +- about.py | 4 +- cmtadm | 14 +- listgetters.py | 3 + sources/antrea.yaml | 12 ++ sources/calico.yaml | 12 ++ sources/cilium.yaml | 12 ++ sources/kubernetes.yaml | 2 +- views/__VersionData.yaml | 79 ++++++++ 10 files changed, 532 insertions(+), 6 deletions(-) create mode 100644 sources/antrea.yaml create mode 100644 sources/calico.yaml create mode 100644 sources/cilium.yaml diff --git a/CHANGELOG/CHANGELOG-0.8.md b/CHANGELOG/CHANGELOG-0.8.md index bb061e1..bb79821 100644 --- a/CHANGELOG/CHANGELOG-0.8.md +++ b/CHANGELOG/CHANGELOG-0.8.md @@ -1,3 +1,399 @@ +* [v0.8.4](#v084) + * [Downloads](#downloads-for-v084) + * [Source Code](#source-code-for-v084) + * [Distro Packages](#distro-packages-for-v084) + * [General Release Notes](#general-release-notes-for-v084) + * [Urgent Upgrade Notes](#urgent-upgrade-notes-for-v084) + * [Changes by Component](#changes-by-component-in-v084) + * [Changes to _cmt_](#changes-to-cmt-in-v084) + * [Changes to _cmtadm_](#changes-to-cmtadm-in-v084) + * [Changes to _cmtinv_](#changes-to-cmtinv-in-v084) + * [Changes to _cmu_](#changes-to-cmu-in-v084) + * [Changes to other files](#changes-to-other-files-in-v084) + * [Fixed Issues](#fixed-issues-in-v084) + * [Known Regressions](#known-regressions-in-v084) + * [Dependencies](#dependencies-for-v084) + * [Test Results](#test-results-for-v084) + * [Bandit](#bandit-results-for-v084) + * [Coverage](#coverage-results-for-v084) + * [Flake8](#flake8-results-for-v084) + * [Pylint](#pylint-results-for-v084) + * [Regexploit](#regexploit-results-for-v084) + * [Ruff](#ruff-results-for-v084) + * [Semgrep](#semgrep-results-for-v084) + * [validate_playbooks](#validate_playbooks-results-for-v084) + * [validate_yaml](#validate_yaml-results-for-v084) + * [YAMLlint](#yamllint-results-for-v084) + +# v0.8.4 + +## Downloads for v0.8.4 + +### Source Code for v0.8.4 + +CMT v0.8.4 does not include source code tarballs. It is just a git tag. +We aim for CMT v0.9.0 to be the first release with source code tarballs. + + + +### Distro packages for v0.8.4 + +CMT v0.8.4 does not include distro packages. It is just a git tag. +We aim for CMT v0.9.0 to be the first release with distro packages. + + + +## General Release Notes for v0.8.4 + +This is a tagged release of __Cluster Management Toolkit for Kubernetes__ (CMT). +It provides support for setting up Kubernetes clusters either using templates (recommended) +or step by step. + +It also provides tools for managing the underlying hosts (and, optionally, hosts +that are not part of the cluster) using Ansible. + +Finally it contains a Curses-based user interface (`cmu`) that provides an overview +of the cluster objects and their relations; for instance the user interface provides +links from the Pod view directly to its controller, config maps, logs, namespace, +secrets, etc. + +## Urgent Upgrade Notes for v0.8.4 + +This release contains another fix for the Kubernetes version check. + +## Changes by Component in v0.8.4 + +### Changes to _cmt_ in v0.8.4 + +No changes. + +### Changes to _cmtadm_ in v0.8.4 + +* `cmtadm check-versions` now has a `--force` option. + +### Changes to _cmtinv_ in v0.8.4 + +No changes. + +### Changes to _cmu_ in v0.8.4 + +No changes. + +### Changes to other files in v0.8.4 + +* `sources/antrea.yaml`, `sources/calico.yaml`, `sources/cilium.yaml`, and `views/__VersionData.yaml`: show version information for antctl, cilium-cli, and calicoctl (if installed). +* `sources/kubernetes.yaml` now uses the correct version check helper. +* `listgetters.py`: listgetter_files() now has an option to skip empty or missing items. + +## Fixed Issues in v0.8.4 + +Fix version check for Kubernetes. + +## Known Regressions in v0.8.4 + +Pylint fails to test the executables; this is an issue in Pylint though, not in CMT. + +## Known Issues in v0.8.4 + +* The UI flickers until data has been populated; it can also flicker in certain other scenarios. +* The version data view in the UI does not refresh the version data. + The version data can be refreshed using `cmtadm cv`. +* Installing to the system path is currently not supported. + +## Dependencies for v0.8.4 + +### Python + +| PIP Name | Minimum Version | Note | +| -------------- | --------------- | --------------------------------------- | +| ansible-runner | 2.1.3 | openSUSE/SLES/RHEL, unsupported distros | +| cryptography | | openSUSE, unsupported distros | +| natsort | 8.0.2 | openSUSE/SLES/RHEL, unsupported distros | +| paramiko | | openSUSE/SLES/RHEL, unsupported distros | +| PyYAML | 6.0 | Unsupported distros | +| ujson | 5.4.0 | openSUSE/SLES/RHEL, unsupported distros | +| urllib3 | 1.26.18 | openSUSE/SLES, unsupported distros | +| validators | 0.22.0 | openSUSE/SLES/RHEL, unsupported distros | + +### Distro Packages + +| Package Name | Distro | +| ---------------------- | ------------------ | +| ansible | Debian/Ubuntu/SUSE | +| python3-ansible-runner | Debian/Ubuntu | +| python3-cryptographya | Debian/RHEL/Ubuntu | +| python3-natsort | Debian/Ubuntu | +| python3-paramiko | Debian/Ubuntu | +| python3-pip | Debian/Ubuntu | +| python3-pyyaml | RHEL | +| python3-ujson | Debian/Ubuntu | +| python3-urllib3 | Debian/Ubuntu/RHEL | +| python3-validators | Debian/Ubuntu | +| python3-yaml | Debian/Ubuntu | +| sshpass | All | + +### Manual Installation or Unknown Distro Packages + +| Software | Distro | +| -------- | ------------------- | +| ansible | Unsupported distros | +| sshpass | Unsupported distros | + +## Test Results for v0.8.4 + +Before release the code quality has been checked with _pylint_, _flake8_, _mypy_, and _ruff_. +The code has been checked for security issues using _bandit_, _regexploit_, and _semgrep_. +The _Ansible_ playbooks have been checked using _ansible-lint_. +YAML-files have been checked using _yamllint_ and validated against predefined schemas. +Unit-test coverage has been measured using _python3-coverage_. + +The results of these tests are as follows: + +### Bandit Results for v0.8.4 + +Commandline: `bandit -c .bandit`. +Execute with `make bandit`. + +Output: + +``` +Test results: + No issues identified. + +Code scanned: + Total lines of code: 71488 + Total lines skipped (#nosec): 15 + +Run metrics: + Total issues (by severity): + Undefined: 0.0 + Low: 0.0 + Medium: 0.0 + High: 0.0 + Total issues (by confidence): + Undefined: 0.0 + Low: 0.0 + Medium: 0.0 + High: 0.0 +Files skipped (0): +``` + +### Coverage Results for v0.8.4 + +Commandline: `python3-coverage run --branch --append && python3-coverage report --sort cover --precision 1`. + +Execute with: + +``` +make coverage +make coverage-ansible +make coverage-cluster +``` + +Output: + +``` +Name Stmts Miss Branch BrPart Cover +-------------------------------------------------------------- +listgetters_async.py 119 100 58 0 10.7% +listgetters.py 1172 959 636 17 15.0% +curses_helper.py 2393 1832 1118 68 19.2% +checks.py 609 457 226 28 19.6% +networkio.py 288 218 149 6 21.1% +itemgetters.py 443 323 254 0 23.8% +logparser.py 1898 1282 1055 30 27.9% +generators.py 740 494 390 5 30.2% +infogetters.py 420 288 208 2 33.4% +kubernetes_helper.py 1433 852 748 103 33.7% +fieldgetters.py 80 39 46 11 41.3% +datagetters.py 274 148 142 11 42.5% +formatters.py 767 324 408 29 54.6% +ansible_helper.py 796 199 488 27 73.8% +cmtlib.py 604 106 346 28 80.2% +cmtio.py 423 35 240 17 91.0% +reexecutor.py 72 2 34 3 95.3% +ansithemeprint.py 211 1 80 3 98.6% +cmtvalidators.py 324 1 200 1 99.6% +commandparser.py 411 0 254 1 99.8% +about.py 19 0 0 0 100.0% +cmtio_yaml.py 34 0 6 0 100.0% +cmtpaths.py 80 0 0 0 100.0% +cmttypes.py 468 0 178 0 100.0% +cni_data.py 38 0 8 0 100.0% +helptexts.py 23 0 0 0 100.0% +kubernetes_resources.py 4 0 0 0 100.0% +objgetters.py 55 0 12 0 100.0% +pvtypes.py 2 0 0 0 100.0% +recommended_permissions.py 11 0 0 0 100.0% +-------------------------------------------------------------- +TOTAL 14211 7660 7284 390 42.8% +``` + + +### Flake8 Results for v0.8.4 + +Commandline: `flake8 --max-line-length 100 --ignore F841,W503 --statistics`. +Execute with `make flake8`. + +Output: + +No output. + +### mypy Results for v0.8.4 + +Commandline: `mypy --ignore-missing --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs --check-untyped-defs --disallow-untyped-decorators`. +Execute with `make mypy-markdown`. + +| Source file | Score | +| ----------------------- | ------------------------------------------------------- | +| cmt | Success: no issues found in 1 source file | +| cmtadm | Success: no issues found in 1 source file | +| cmt-install | Success: no issues found in 1 source file | +| cmtinv | Success: no issues found in 1 source file | +| cmu | __Found 545 errors in 2 files (checked 1 source file)__ | +| about.py | Success: no issues found in 1 source file | +| ansible_helper.py | Success: no issues found in 1 source file | +| ansithemeprint.py | Success: no issues found in 1 source file | +| checks.py | Success: no issues found in 1 source file | +| cmtio.py | Success: no issues found in 1 source file | +| cmtio_yaml.py | Success: no issues found in 1 source file | +| cmtlib.py | Success: no issues found in 1 source file | +| cmtpaths.py | Success: no issues found in 1 source file | +| cmttypes.py | Success: no issues found in 1 source file | +| cmtvalidators.py | Success: no issues found in 1 source file | +| cni_data.py | Success: no issues found in 1 source file | +| commandparser.py | Success: no issues found in 1 source file | +| curses_helper.py | Success: no issues found in 1 source file | +| datagetters.py | Success: no issues found in 1 source file | +| fieldgetters.py | Success: no issues found in 1 source file | +| formatters.py | Success: no issues found in 1 source file | +| generators.py | Success: no issues found in 1 source file | +| helptexts.py | Success: no issues found in 1 source file | +| infogetters.py | Success: no issues found in 1 source file | +| itemgetters.py | Success: no issues found in 1 source file | +| kubernetes_helper.py | Success: no issues found in 1 source file | +| kubernetes_resources.py | Success: no issues found in 1 source file | +| listgetters.py | Success: no issues found in 1 source file | +| listgetters_async.py | Success: no issues found in 1 source file | +| logparser.py | __Found 103 errors in 1 file (checked 1 source file)__ | +| networkio.py | Success: no issues found in 1 source file | +| objgetters.py | Success: no issues found in 1 source file | +| pvtypes.py | Success: no issues found in 1 source file | +| reexecutor.py | Success: no issues found in 1 source file | + +### Pylint Results for v0.8.4 + +**N/A**. Pylint seems to be broken in Debian at the moment and crashes during testing. + +### Regexploit Results for v0.8.4 + +Commandline: `regexploit`. +Execute with `make regexploit`. + +Output: + +``` +Running regexploit to check for ReDoS attacks + +Checking executables +Processed 44 regexes + +Checking libraries +Processed 117 regexes +``` + +### Ruff Results for v0.8.4 + +Commandline: `ruff`. +Execute with `make ruff`. + +Output: + +No output. + +### Semgrep Results for v0.8.4 + +Commandline: `semgrep scan --exclude-rule "generic.secrets.security.detected-generic-secret.detected-generic-secret.semgrep-legacy.30980" --timeout=0 --no-git-ignore`. +Execute with `make semgrep`. + +Output: + +``` +┌──── ○○○ ────┐ +│ Semgrep CLI │ +└─────────────┘ + +Scanning 911 files with: + +✔ Semgrep OSS + ✔ Basic security coverage for first-party code vulnerabilities. + +✔ Semgrep Code (SAST) + ✔ Find and fix vulnerabilities in the code you write with advanced scanning and expert security rules. + +✘ Semgrep Supply Chain (SCA) + ✘ Find and fix the reachable vulnerabilities in your OSS dependencies. + + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╸ 100% 0:00:38 + + +┌──────────────┐ +│ Scan Summary │ +└──────────────┘ +Some files were skipped or only partially analyzed. + Partially scanned: 2 files only partially analyzed due to parsing or internal Semgrep errors + Scan skipped: 73 files matching .semgrepignore patterns + For a full list of skipped files, run semgrep with the --verbose flag. + +Ran 472 rules on 838 files: 0 findings. + +✨ If Semgrep missed a finding, please send us feedback to let us know! + See https://semgrep.dev/docs/reporting-false-negatives/ +``` + +### validate_playbooks Results for v0.8.4 + +Commandline: `ansible-lint`. +Execute with: `make validate_playbooks`. + +`Passed: 0 failure(s), 0 warning(s) on 55 files. Last profile that met the validation criteria was 'production'.` + +### validate_yaml Results for v0.8.4 + +Commandline: `tests/validate_yaml`. +Execute with: `make validate_yaml`. + +Output: + +``` +Summary: + fail: 0 + skip: 1 + success: 641 + total: 642 +``` + +### YAMLlint Results for v0.8.4 + +Commandline: `yamllint`. +Execute with `make yamllint`. + +Output: + +No output. + +--- + * [v0.8.3](#v083) * [Downloads](#downloads-for-v083) * [Source Code](#source-code-for-v083) diff --git a/CHANGELOG/latest.yaml b/CHANGELOG/latest.yaml index 6f8c890..3806fc3 100644 --- a/CHANGELOG/latest.yaml +++ b/CHANGELOG/latest.yaml @@ -1,3 +1,3 @@ -release: "0.8.2" -release_date: "2024-04-14" +release: "0.8.4" +release_date: "2024-04-18" changelog: "CHANGELOG-0.8.md" diff --git a/about.py b/about.py index 401793a..b342bf3 100644 --- a/about.py +++ b/about.py @@ -17,7 +17,7 @@ PROGRAM_SUITE_NAME = "CMT" PROGRAM_SUITE_FULL_NAME = "Cluster Management Toolkit for Kubernetes" -PROGRAM_SUITE_VERSION = "0.8.3" +PROGRAM_SUITE_VERSION = "0.8.4" UI_PROGRAM_NAME = "cmu" UI_PROGRAM_VERSION = "0.5.1" @@ -29,7 +29,7 @@ INSTALL_PROGRAM_VERSION = "0.13.6" ADMIN_PROGRAM_NAME = "cmtadm" -ADMIN_PROGRAM_VERSION = "0.9.1" +ADMIN_PROGRAM_VERSION = "0.9.2" INVENTORY_PROGRAM_NAME = "cmtinv" INVENTORY_PROGRAM_VERSION = "0.4.5" diff --git a/cmtadm b/cmtadm index 4227a96..62e7678 100755 --- a/cmtadm +++ b/cmtadm @@ -3639,8 +3639,11 @@ def check_for_updates(options: List[Tuple[str, str]], args: List[str]) -> None: # default options update_cache = True verbose = False + force = False for opt, _optarg in options: + if opt == "--force": + force = True if opt == "--no-cache-update": update_cache = False elif opt == "--verbose": @@ -3661,7 +3664,7 @@ def check_for_updates(options: List[Tuple[str, str]], args: List[str]) -> None: check_and_print_status(update_zypper_cache()) ansithemeprint([ANSIThemeStr("\n• ", "separator"), ANSIThemeStr("Updating version cache", "action")]) - update_version_cache(verbose=verbose) + update_version_cache(verbose=verbose, force=force) pkg_packages = [ "ansible", @@ -6581,6 +6584,15 @@ COMMANDLINE = { [ANSIThemeStr("relevant to the configuration in use", "description")], ], "options": { + "--force": { + "description": [ + ANSIThemeStr("Force update of the version cache", "description")], + "extended_description": [ + [ANSIThemeStr("Typically version cache updates are", "description")], + [ANSIThemeStr("rate-limited to once per hour; this option", "description")], + [ANSIThemeStr("forces an update during the cooldown period.", "description")], + ], + }, "--no-cache-update": { "description": [ ANSIThemeStr("Do not update the APT cache", "description")], diff --git a/listgetters.py b/listgetters.py index 66a8932..03f7d8b 100644 --- a/listgetters.py +++ b/listgetters.py @@ -385,6 +385,7 @@ def listgetter_files(**kwargs: Any) -> Tuple[List[Dict[str, Any]], Union[None, i [DictPath("file_not_found_status"), DictPath("extra_values#_extra_data#file_not_found_status")], "File not found", fallback_on_empty=True) + skip_empty_paths = deep_get(kwargs, DictPath("skip_empty_paths"), False) vlist: List[Dict[str, Any]] = [] status = None @@ -447,6 +448,8 @@ def listgetter_files(**kwargs: Any) -> Tuple[List[Dict[str, Any]], Union[None, i item = d else: item = __recurse_data(path, d) + if (item is None or not item) and skip_empty_paths: + continue extra_data = deep_get(path, DictPath("_extra_data"), {}) extra_data["mtime"] = mtime diff --git a/sources/antrea.yaml b/sources/antrea.yaml new file mode 100644 index 0000000..7820906 --- /dev/null +++ b/sources/antrea.yaml @@ -0,0 +1,12 @@ +antctl: + description: "antctl (Antrea)" + installed_version: + function: "get_executable_version" + executable: "/usr/bin/antctl" + args: ["version"] + version_regex: '^antctlVersion: v(\d+)(\.)(\d+)(\.)(\d+)' + candidate_version: + function: "get_github_version" + args: + url: "https://api.github.com/repos/antrea-io/antrea/releases" + version_regex: 'v(\d+)(\.)(\d+)(\.)(\d+)$' diff --git a/sources/calico.yaml b/sources/calico.yaml new file mode 100644 index 0000000..426ce41 --- /dev/null +++ b/sources/calico.yaml @@ -0,0 +1,12 @@ +calicoctl: + description: "calicoctl (Calico)" + installed_version: + function: "get_executable_version" + executable: "/usr/bin/calicoctl" + args: ["version"] + version_regex: '^Client Version:\s+v(\d+)(\.)(\d+)(\.)(\d+)' + candidate_version: + function: "get_github_version" + args: + url: "https://api.github.com/repos/projectcalico/calico/releases" + version_regex: 'v(\d+)(\.)(\d+)(\.)(\d+)$' diff --git a/sources/cilium.yaml b/sources/cilium.yaml new file mode 100644 index 0000000..e060341 --- /dev/null +++ b/sources/cilium.yaml @@ -0,0 +1,12 @@ +cilium-cli: + description: "cilium-cli (Calico)" + installed_version: + function: "get_executable_version" + executable: "/usr/bin/cilium" + args: ["version"] + version_regex: '^cilium-cli:\s+v(\d+)(\.)(\d+)(\.)(\d+).*' + candidate_version: + function: "get_github_version" + args: + url: "https://api.github.com/repos/cilium/cilium-cli/releases" + version_regex: 'v(\d+)(\.)(\d+)(\.)(\d+)$' diff --git a/sources/kubernetes.yaml b/sources/kubernetes.yaml index 0827348..2f40882 100644 --- a/sources/kubernetes.yaml +++ b/sources/kubernetes.yaml @@ -1,7 +1,7 @@ kubernetes: description: "Kubernetes" candidate_version: - function: "get_github_tag_version" + function: "get_github_version" args: url: "https://api.github.com/repos/kubernetes/kubernetes/releases" version_regex: 'v(\d+)(\.)(\d+)(\.)(\d+)$' diff --git a/views/__VersionData.yaml b/views/__VersionData.yaml index c575da1..3f32dbf 100644 --- a/views/__VersionData.yaml +++ b/views/__VersionData.yaml @@ -10,6 +10,7 @@ listview: listgetter_args: # We should call update_version_cache() here file_not_found_status: "No version data found" + skip_empty_paths: true paths: - filepath: "{HOME}/cluster-management-toolkit/CHANGELOG/latest.yaml" filetype: "yaml" @@ -103,6 +104,84 @@ listview: include_end: false strip_empty_start: false strip_empty_end: true + - filepath: "{HOME}/.cmt/version-cache/candidate_versions.yaml" + filetype: "yaml" + path: "antctl" + pathtype: "dict" + _extra_data: + name: "antctl (Antrea)" + installed_version: + fieldgetter: "fieldgetter_executable_version" + fieldgetter_args: + executable: "/usr/bin/antctl" + args: ["version"] + version_regex: '^antctlVersion: v(\d+)(\.)(\d+)(\.)(\d+)' + file_not_found_status: "No changelog found" + paths: + - filepath: "{HOME}/.cmt/version-cache/antrea_current_changelog.md" + filetype: "text" + pathtype: "raw" + formatter: "markdown" + formatter_args: + # First match in the file is what we want + start: "## Changelog since " + include_start: true + end: "# v" + include_end: false + strip_empty_start: false + strip_empty_end: true + - filepath: "{HOME}/.cmt/version-cache/candidate_versions.yaml" + filetype: "yaml" + path: "calicoctl" + pathtype: "dict" + _extra_data: + name: "calicoctl (Calico)" + installed_version: + fieldgetter: "fieldgetter_executable_version" + fieldgetter_args: + executable: "/usr/bin/calicoctl" + args: ["version"] + version_regex: '^Client Version:\s+v(\d+)(\.)(\d+)(\.)(\d+)' + file_not_found_status: "No changelog found" + paths: + - filepath: "{HOME}/.cmt/version-cache/calico_current_changelog.md" + filetype: "text" + pathtype: "raw" + formatter: "markdown" + formatter_args: + # First match in the file is what we want + start: "## Changelog since " + include_start: true + end: "# v" + include_end: false + strip_empty_start: false + strip_empty_end: true + - filepath: "{HOME}/.cmt/version-cache/candidate_versions.yaml" + filetype: "yaml" + path: "cilium-cli" + pathtype: "dict" + _extra_data: + name: "cilium-cli (Cilium)" + installed_version: + fieldgetter: "fieldgetter_executable_version" + fieldgetter_args: + executable: "/usr/bin/cilium" + args: ["version"] + version_regex: '^cilium-cli:\s+v(\d+)(\.)(\d+)(\.)(\d+).*' + file_not_found_status: "No changelog found" + paths: + - filepath: "{HOME}/.cmt/version-cache/cilium_current_changelog.md" + filetype: "text" + pathtype: "raw" + formatter: "markdown" + formatter_args: + # First match in the file is what we want + start: "## Changelog since " + include_start: true + end: "# v" + include_end: false + strip_empty_start: false + strip_empty_end: true infogetter: "generic_infogetter" field_indexes: Normal: