From 49680cf0817c8fb452695aa3d28ff7f1a3dff797 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Mon, 20 Mar 2023 12:21:24 +0100 Subject: [PATCH] CI: Add ansible-test run --- .github/workflows/test.yml | 33 +++++++- .gitignore | 1 + plugins/module_utils/cephadm_common.py | 7 +- plugins/modules/cephadm_crush_rule.py | 35 ++++---- plugins/modules/cephadm_ec_profile.py | 48 ++++++----- plugins/modules/cephadm_key.py | 106 ++++++++++++++++++------- plugins/modules/cephadm_pool.py | 68 ++++++++++------ test-requirements.txt | 1 - tests/sanity/ignore-2.14.txt | 13 +++ 9 files changed, 217 insertions(+), 95 deletions(-) create mode 100644 tests/sanity/ignore-2.14.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4160d42..859d121 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,4 +26,35 @@ jobs: run: | ansible-lint -v --force-color antsibull-changelog lint - flake8 -v + + ansible-test-sanity: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + # Checks-out the repository under $GITHUB_WORKSPACE, so it's accessible to the job + - uses: actions/checkout@v3 + with: + path: ansible_collections/stackhpc/cephadm + + - name: Install dependencies + working-directory: ansible_collections/stackhpc/cephadm + run: | + python -m pip install --upgrade pip + pip install -r test-requirements.txt + + - name: Run ansible-test sanity to download docker images + working-directory: ansible_collections/stackhpc/cephadm + run: | + ansible-test sanity --docker --prime-containers + + - name: Run ansible-test sanity + working-directory: ansible_collections/stackhpc/cephadm + run: | + ansible-test sanity --verbose --docker --junit + + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + if: success() || failure() # always run even if the previous step fails + with: + report_paths: '**/tests/output/junit/*.xml' diff --git a/.gitignore b/.gitignore index 6a1fba6..5212215 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /changelogs/.plugin-cache.yaml +/tests/output diff --git a/plugins/module_utils/cephadm_common.py b/plugins/module_utils/cephadm_common.py index e3abdb7..3d1f42a 100644 --- a/plugins/module_utils/cephadm_common.py +++ b/plugins/module_utils/cephadm_common.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 - # Copyright 2020, Red Hat, Inc. # Copyright 2021, StackHPC, Ltd. # NOTE: Files adapted from github.com/ceph/ceph-ansible @@ -16,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + import datetime @@ -75,4 +76,4 @@ def fatal(message, module): if module: module.fail_json(msg=message, rc=1) else: - raise(Exception(message)) + raise Exception(message) diff --git a/plugins/modules/cephadm_crush_rule.py b/plugins/modules/cephadm_crush_rule.py index bf3e360..2da8a48 100644 --- a/plugins/modules/cephadm_crush_rule.py +++ b/plugins/modules/cephadm_crush_rule.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python # Copyright 2020, Red Hat, Inc. # Copyright 2021, StackHPC, Ltd. @@ -19,13 +19,6 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -from ansible.module_utils.basic import AnsibleModule -from ansible_collections.stackhpc.cephadm.plugins.module_utils.cephadm_common \ - import generate_ceph_cmd, exec_command, exit_module - -import datetime -import json - DOCUMENTATION = r''' --- @@ -34,11 +27,15 @@ version_added: "1.4.0" description: - Manage Ceph Crush rule(s) creation, deletion and updates. +author: + - Dimitri Savineau + - Michal Nasiadka options: name: description: - name of the Ceph Crush rule. required: true + type: str state: description: If 'present' is used, the module creates a rule if it doesn't @@ -49,33 +46,35 @@ required: false choices: ['present', 'absent', 'info'] default: present + type: str rule_type: description: - The ceph CRUSH rule type. required: false choices: ['replicated', 'erasure'] - required: false + type: str bucket_root: description: - The ceph bucket root for replicated rule. required: false + type: str bucket_type: description: - The ceph bucket type for replicated rule. required: false choices: ['osd', 'host', 'chassis', 'rack', 'row', 'pdu', 'pod', 'room', 'datacenter', 'zone', 'region', 'root'] + type: str device_class: description: - The ceph device class for replicated rule. required: false + type: str profile: description: - The ceph erasure profile for erasure rule. required: false -author: - - Dimitri Savineau - Michal Nasiadka + type: str ''' EXAMPLES = ''' @@ -107,6 +106,14 @@ RETURN = '''# ''' +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.stackhpc.cephadm.plugins.module_utils.cephadm_common \ + import generate_ceph_cmd, exec_command, exit_module + +import datetime +import json + + def create_rule(module, container_image=None): ''' Create a new crush replicated/erasure rule @@ -211,7 +218,7 @@ def main(): else: rule = json.loads(out) if (rule['type'] == 1 and rule_type == 'erasure') or (rule['type'] == 3 and rule_type == 'replicated'): # noqa: E501 - module.fail_json(msg="Can not convert crush rule {} to {}".format(name, rule_type), changed=False, rc=1) # noqa: E501 + module.fail_json(msg="Can not convert crush rule {0} to {1}".format(str(name), str(rule_type)), changed=False, rc=1) # noqa: E501 elif state == "absent": rc, cmd, out, err = exec_command(module, get_rule(module)) # noqa: E501 @@ -220,7 +227,7 @@ def main(): changed = True else: rc = 0 - out = "Crush Rule {} doesn't exist".format(name) + out = "Crush Rule {0} doesn't exist".format(name) elif state == "info": rc, cmd, out, err = exec_command(module, get_rule(module)) # noqa: E501 diff --git a/plugins/modules/cephadm_ec_profile.py b/plugins/modules/cephadm_ec_profile.py index 56aee0d..2554e9b 100644 --- a/plugins/modules/cephadm_ec_profile.py +++ b/plugins/modules/cephadm_ec_profile.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python # Copyright 2020, Red Hat, Inc. # Copyright 2021, StackHPC, Ltd. @@ -19,13 +19,6 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -from ansible.module_utils.basic import AnsibleModule -from ansible_collections.stackhpc.cephadm.plugins.module_utils.cephadm_common \ - import generate_ceph_cmd, exec_command, exit_module - -import datetime -import json - DOCUMENTATION = ''' --- @@ -42,49 +35,52 @@ description: - name of the profile. required: true + type: str state: description: If 'present' is used, the module creates a profile. If 'absent' is used, the module will delete the profile. required: false - choices: ['present', 'absent', 'info'] + choices: ['present', 'absent'] default: present + type: str stripe_unit: description: - The amount of data in a data chunk, per stripe. required: false + type: str k: description: - Number of data-chunks the object will be split in required: false + type: str m: description: - Compute coding chunks for each object and store them on different OSDs. required: false + type: str plugin: description: - Use the erasure code plugin to compute coding chunks and recover missing chunks. required: false + type: str directory: description: - Set the directory name from which the erasure code plugin is loaded. required: false - crush_root: - description: - - The name of the crush bucket used for the first step of the CRUSH - rule. - required: false + type: str crush_device_class: description: - Restrict placement to devices of a specific class (hdd/ssd) required: false + type: str author: - Guillaume Abrioux - Michal Nasiadka + - Michal Nasiadka ''' EXAMPLES = ''' @@ -100,6 +96,14 @@ state: absent ''' +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.stackhpc.cephadm.plugins.module_utils.cephadm_common \ + import generate_ceph_cmd, exec_command, exit_module + +import datetime +import json + + RETURN = '''# ''' @@ -121,15 +125,15 @@ def create_profile(module, name, k, m, stripe_unit, crush_device_class, director Create a profile ''' - args = ['set', name, 'k={}'.format(k), 'm={}'.format(m)] + args = ['set', name, 'k={0}'.format(k), 'm={0}'.format(m)] if stripe_unit: - args.append('stripe_unit={}'.format(stripe_unit)) + args.append('stripe_unit={0}'.format(stripe_unit)) if crush_device_class: - args.append('crush-device-class={}'.format(crush_device_class)) + args.append('crush-device-class={0}'.format(crush_device_class)) if directory: - args.append('directory={}'.format(plugin)) + args.append('directory={0}'.format(plugin)) if plugin: - args.append('plugin={}'.format(plugin)) + args.append('plugin={0}'.format(plugin)) if force: args.append('--force') @@ -234,11 +238,11 @@ def run_module(): elif state == "absent": rc, cmd, out, err = exec_command(module, delete_profile(module, name)) # noqa: E501 if not err: - out = 'Profile {} removed.'.format(name) + out = 'Profile {0} removed.'.format(name) changed = True else: rc = 0 - out = "Skipping, the profile {} doesn't exist".format(name) + out = "Skipping, the profile {0} doesn't exist".format(name) exit_module(module=module, out=out, rc=rc, cmd=cmd, err=err, startd=startd, changed=changed) # noqa: E501 diff --git a/plugins/modules/cephadm_key.py b/plugins/modules/cephadm_key.py index d8c10cf..d9fcdc2 100644 --- a/plugins/modules/cephadm_key.py +++ b/plugins/modules/cephadm_key.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python # Copyright 2018, Red Hat, Inc. # Copyright 2021, StackHPC, Ltd. @@ -19,16 +19,6 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -from ansible.module_utils.basic import AnsibleModule -from ansible_collections.stackhpc.cephadm.plugins.module_utils.cephadm_common \ - import fatal, generate_ceph_cmd -import datetime -import json -import os -import struct -import time -import base64 - DOCUMENTATION = r''' --- module: cephadm_key @@ -46,7 +36,8 @@ name: description: - name of the CephX key - required: true + required: false + type: str state: description: - If 'present' is used, the module creates a keyring @@ -64,19 +55,27 @@ cephx keyring. required: false choices: ['present', 'update', 'absent', 'list', 'info', - 'generate_secret'] + 'fetch_initial_keys', 'generate_secret'] default: present + type: str caps: description: - CephX key capabilities - default: None + default: {} required: false + type: dict secret: description: - keyring's secret value required: false - default: None - + default: '' + type: str + dest: + description: + - destination directory to save key + required: false + default: "/etc/ceph/" + type: str import_key: description: - Whether or not to import the created keyring into Ceph. @@ -84,25 +83,66 @@ keyrings but not add them into Ceph. required: false default: True + type: bool output_format: description: - The key output format when retrieving the information of an entity. required: false + choices: ['json', 'plain', 'xml', 'yaml'] default: json + type: str + attributes: + aliases: + - attr + description: + - File attributes + required: false + type: str + group: + description: + - Group name for file ownership + required: false + type: str + mode: + description: + - File permission mode + required: false + type: raw + owner: + description: + - File owner + required: false + type: str + selevel: + description: + - SELinux level + required: false + type: str + serole: + description: + - SELinux role + required: false + type: str + setype: + description: + - SELinux type + required: false + type: str + seuser: + description: + - SELinux user + required: false + type: str + unsafe_writes: + description: + - Enable unsafe writes + required: false + default: false + type: bool ''' EXAMPLES = ''' -keys_to_create: - - name: client.key - secret: "AQAin8tUUK84ExAA/QgBtI7gEMWdmnvKBzlXdQ==" - caps: - mon: "allow rwx" - mds: "allow *" - - name: client.cle - caps: - mon: "allow r", osd: "allow *" - - name: create cephx key ceph_key: name: "{{ item.name }}" @@ -142,6 +182,16 @@ RETURN = r''' # ''' +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.stackhpc.cephadm.plugins.module_utils.cephadm_common \ + import fatal, generate_ceph_cmd +import datetime +import json +import os +import struct +import time +import base64 + def str_to_bool(val): try: @@ -329,8 +379,8 @@ def run_module(): name=dict(type='str', required=False), state=dict(type='str', required=False, default='present', choices=['present', 'update', 'absent', # noqa: E501 'list', 'info', 'fetch_initial_keys', 'generate_secret']), # noqa: E501 - caps=dict(type='dict', required=False, default=None), - secret=dict(type='str', required=False, default=None, no_log=True), + caps=dict(type='dict', required=False, default={}), + secret=dict(type='str', required=False, default='', no_log=True), import_key=dict(type='bool', required=False, default=True), dest=dict(type='str', required=False, default='/etc/ceph/'), output_format=dict(type='str', required=False, default='json', choices=['json', 'plain', 'xml', 'yaml']) # noqa: E501 diff --git a/plugins/modules/cephadm_pool.py b/plugins/modules/cephadm_pool.py index 332d54c..9774bbe 100644 --- a/plugins/modules/cephadm_pool.py +++ b/plugins/modules/cephadm_pool.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python # Copyright 2020, Red Hat, Inc. # Copyright 2021, StackHPC, Ltd. @@ -19,17 +19,11 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -from ansible.module_utils.basic import AnsibleModule -from ansible_collections.stackhpc.cephadm.plugins.module_utils.cephadm_common \ - import generate_ceph_cmd, exec_command, exit_module - -import datetime -import json - DOCUMENTATION = r''' module: cephadm_pool -author: Guillaume Abrioux - Michal Nasiadka +author: + - Guillaume Abrioux + - Michal Nasiadka short_description: Manage Ceph Pools version_added: "1.4.0" description: @@ -39,6 +33,7 @@ description: - name of the Ceph pool required: true + type: str state: description: - If 'present' is used, the module creates a pool if it doesn't @@ -49,71 +44,85 @@ required: false choices: ['present', 'absent', 'list'] default: present + type: str + details: + description: + - show details when state is list + required: false + type: bool size: description: - set the replica size of the pool. required: false - default: 3 + type: str min_size: description: - set the min_size parameter of the pool. required: false default: default to `osd_pool_default_min_size` (ceph) + type: str pg_num: description: - set the pg_num of the pool. required: false default: default to `osd_pool_default_pg_num` (ceph) + type: str pgp_num: description: - set the pgp_num of the pool. required: false default: default to `osd_pool_default_pgp_num` (ceph) + type: str pg_autoscale_mode: description: - set the pg autoscaler on the pool. required: false default: 'on' + type: str target_size_ratio: description: - set the target_size_ratio on the pool required: false - default: None + type: str pool_type: description: - set the pool type, either 'replicated' or 'erasure' required: false default: replicated + choices: ['replicated', 'erasure'] + type: str erasure_profile: description: - When pool_type = 'erasure', set the erasure profile of the pool required: false default: default + type: str rule_name: description: - Set the crush rule name assigned to the pool required: false - default: replicated_rule␣when␣pool_type␣is␣erasure␣else␣none + type: str expected_num_objects: description: - Set the expected_num_objects parameter of the pool. required: false - default: 0 + default: "0" + type: str application: description: - Set the pool application on the pool. required: false default: None + type: str + allow_ec_overwrites: + description: + - Set the allow_ec_overwrites paramter of the pool. + required: false + default: false + type: bool ''' EXAMPLES = r''' -pools: - - name: foo - size: 3 - application: rbd - pool_type: 'replicated' - pg_autoscale_mode: 'on - - hosts: all become: true tasks: @@ -130,6 +139,13 @@ RETURN = r'''# ''' +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.stackhpc.cephadm.plugins.module_utils.cephadm_common \ + import generate_ceph_cmd, exec_command, exit_module + +import datetime +import json + def check_pool_exist(name, output_format='json'): @@ -412,7 +428,7 @@ def update_pool(module, name, delta): if rc != 0: return rc, cmd, out, err - report = report + "\n" + "{} has been updated: {} is now {}".format(name, key, delta[key]['value']) # noqa: E501 + report = report + "\n" + "{0} has been updated: {1} is now {2}".format(name, key, delta[key]['value']) # noqa: E501 out = report return rc, cmd, out, err @@ -431,7 +447,7 @@ def run_module(): pg_autoscale_mode=dict(type='str', required=False, default='on'), target_size_ratio=dict(type='str', required=False, default=None), pool_type=dict(type='str', required=False, default='replicated', - choices=['replicated', 'erasure', '1', '3']), + choices=['replicated', 'erasure']), erasure_profile=dict(type='str', required=False, default='default'), rule_name=dict(type='str', required=False, default=None), expected_num_objects=dict(type='str', required=False, default="0"), @@ -543,7 +559,7 @@ def run_module(): delta.pop('pgp_num', None) if len(delta) == 0: - out = "Skipping pool {}.\nUpdating either 'size' on an erasure-coded pool or 'pg_num'/'pgp_num' on a pg autoscaled pool is incompatible".format(name) # noqa: E501 + out = "Skipping pool {0}.\nUpdating either 'size' on an erasure-coded pool or 'pg_num'/'pgp_num' on a pg autoscaled pool is incompatible".format(name) # noqa: E501 else: rc, cmd, out, err = update_pool(module, name, @@ -552,7 +568,7 @@ def run_module(): changed = True else: - out = "Pool {} already exists and there is nothing to update.".format(name) # noqa: E501 + out = "Pool {0} already exists and there is nothing to update.".format(name) # noqa: E501 else: rc, cmd, out, err = exec_command(module, create_pool(name, @@ -586,7 +602,7 @@ def run_module(): changed = True else: rc = 0 - out = "Skipped, since pool {} doesn't exist".format(name) + out = "Skipped, since pool {0} doesn't exist".format(name) exit_module(module=module, out=out, rc=rc, cmd=cmd, err=err, startd=startd, changed=changed) diff --git a/test-requirements.txt b/test-requirements.txt index 5b93992..c4d0a31 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,3 @@ ansible>=2.9 ansible-lint<7 antsibull-changelog -flake8<5 diff --git a/tests/sanity/ignore-2.14.txt b/tests/sanity/ignore-2.14.txt new file mode 100644 index 0000000..9a56b7c --- /dev/null +++ b/tests/sanity/ignore-2.14.txt @@ -0,0 +1,13 @@ +plugins/modules/cephadm_pool.py pylint:disallowed-name +plugins/modules/cephadm_crush_rule.py validate-modules:missing-gplv3-license +plugins/modules/cephadm_crush_rule.py validate-modules:parameter-state-invalid-choice +plugins/modules/cephadm_crush_rule.py validate-modules:invalid-documentation +plugins/modules/cephadm_ec_profile.py validate-modules:invalid-documentation +plugins/modules/cephadm_ec_profile.py validate-modules:missing-gplv3-license +plugins/modules/cephadm_key.py validate-modules:invalid-documentation +plugins/modules/cephadm_key.py validate-modules:missing-gplv3-license +plugins/modules/cephadm_key.py validate-modules:parameter-state-invalid-choice +plugins/modules/cephadm_pool.py validate-modules:missing-gplv3-license +plugins/modules/cephadm_pool.py validate-modules:parameter-state-invalid-choice +plugins/modules/cephadm_pool.py validate-modules:invalid-documentation +plugins/modules/cephadm_pool.py validate-modules:doc-default-does-not-match-spec