Skip to content

Commit

Permalink
ceph-volume: update functional testing
Browse files Browse the repository at this point in the history
various changes for fixing ceph-volume functional testing.
cleaning up deploy.yml (drop py2 references)

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
(cherry picked from commit 90bfaaa)
(cherry picked from commit bf448e3)
  • Loading branch information
guits committed Apr 12, 2024
1 parent f8948b5 commit 63fe392
Show file tree
Hide file tree
Showing 88 changed files with 69 additions and 574 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,29 +14,29 @@
tasks:

- name: mark osds down
command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
command: "ceph osd down osd.{{ item }}"
with_items: "{{ osd_ids }}"
- name: purge osds
command: "ceph --cluster {{ cluster }} osd purge osd.{{ item }} --yes-i-really-mean-it"
command: "ceph osd purge osd.{{ item }} --yes-i-really-mean-it"
with_items: "{{ osd_ids }}"

- hosts: osds
become: yes
tasks:

- name: zap devices used for OSDs
command: "ceph-volume --cluster {{ cluster }} lvm zap {{ item }} --destroy"
command: "ceph-volume lvm zap {{ item }} --destroy"
with_items: "{{ devices }}"
environment:
CEPH_VOLUME_DEBUG: 1

- name: batch create devices again
command: "ceph-volume --cluster {{ cluster }} lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}"
command: "ceph-volume lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}"
environment:
CEPH_VOLUME_DEBUG: 1

- name: ensure batch create is idempotent
command: "ceph-volume --cluster {{ cluster }} lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}"
command: "ceph-volume lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}"
register: batch_cmd
failed_when: false
environment:
Expand All @@ -50,7 +50,7 @@
- "'strategy changed' not in batch_cmd.stderr"

- name: run batch --report to see if devices get filtered
command: "ceph-volume --cluster {{ cluster }} lvm batch --report --format=json --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}"
command: "ceph-volume lvm batch --report --format=json --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}"
register: report_cmd
failed_when: false
environment:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@
tasks:

- name: mark osds down
command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
command: "ceph osd down osd.{{ item }}"
with_items: "{{ osd_ids }}"
- name: purge osds
command: "ceph --cluster {{ cluster }} osd purge osd.{{ item }} --yes-i-really-mean-it"
command: "ceph osd purge osd.{{ item }} --yes-i-really-mean-it"
with_items: "{{ osd_ids }}"

- hosts: osds
Expand All @@ -27,18 +27,18 @@
tasks:

- name: zap devices used for OSDs
command: "ceph-volume --cluster {{ cluster }} lvm zap {{ item }} --destroy"
command: "ceph-volume lvm zap {{ item }} --destroy"
with_items: "{{ devices }}"
environment:
CEPH_VOLUME_DEBUG: 1

- name: batch create devices again
command: "ceph-volume --cluster {{ cluster }} lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} {{ external_devices }} {{ devices[2:] | join(' ') }}"
command: "ceph-volume lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} {{ external_devices }} {{ devices[2:] | join(' ') }}"
environment:
CEPH_VOLUME_DEBUG: 1

- name: ensure batch create is idempotent when all data devices are filtered
command: "ceph-volume --cluster {{ cluster }} lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} {{ external_devices }} {{ devices[2:] | join(' ') }}"
command: "ceph-volume lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} {{ external_devices }} {{ devices[2:] | join(' ') }}"
register: batch_cmd
failed_when: false
environment:
Expand All @@ -51,7 +51,7 @@
- batch_cmd.rc != 0

- name: run batch --report to see if devices get filtered
command: "ceph-volume --cluster {{ cluster }} lvm batch --report --format=json --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} {{ external_devices }} {{ devices[2:] | join(' ') }}"
command: "ceph-volume lvm batch --report --format=json --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} {{ external_devices }} {{ devices[2:] | join(' ') }}"
register: report_cmd
failed_when: false
environment:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
tasks:

- name: mark osds down
command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
command: "ceph osd down osd.{{ item }}"
with_items: "{{ osd_ids }}"
- name: purge osds
command: "ceph --cluster {{ cluster }} osd purge osd.{{ item }} --yes-i-really-mean-it"
command: "ceph osd purge osd.{{ item }} --yes-i-really-mean-it"
with_items: "{{ osd_ids }}"


Expand All @@ -27,7 +27,7 @@
tasks:

- name: zap devices used for OSDs
command: "ceph-volume --cluster {{ cluster }} lvm zap --osd-id {{ item }} --destroy"
command: "ceph-volume lvm zap --osd-id {{ item }} --destroy"
with_items: "{{ osd_ids }}"
environment:
CEPH_VOLUME_DEBUG: 1
Expand Down
21 changes: 10 additions & 11 deletions src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[tox]
envlist = centos8-bluestore-{single_type,single_type_dmcrypt,mixed_type,mixed_type_dmcrypt,mixed_type_explicit,mixed_type_dmcrypt_explicit}
envlist = centos-bluestore-{single_type,single_type_dmcrypt,mixed_type,mixed_type_dmcrypt,mixed_type_explicit,mixed_type_dmcrypt_explicit}
skipsdist = True

[testenv]
Expand All @@ -18,17 +18,19 @@ setenv=
VAGRANT_CWD = {changedir}
CEPH_VOLUME_DEBUG = 1
DEBIAN_FRONTEND=noninteractive
ANSIBLE_COLLECTIONS_PATH = {envdir}/ansible_collections
CEPH_ANSIBLE_VAGRANT_BOX = centos/stream8
changedir=
centos8-bluestore-single_type: {toxinidir}/centos8/bluestore/single-type
centos8-bluestore-single_type_dmcrypt: {toxinidir}/centos8/bluestore/single-type-dmcrypt
centos8-bluestore-mixed_type: {toxinidir}/centos8/bluestore/mixed-type
centos8-bluestore-mixed_type_dmcrypt: {toxinidir}/centos8/bluestore/mixed-type-dmcrypt
centos8-bluestore-mixed_type_explicit: {toxinidir}/centos8/bluestore/mixed-type-explicit
centos8-bluestore-mixed_type_dmcrypt_explicit: {toxinidir}/centos8/bluestore/mixed-type-dmcrypt-explicit
centos-bluestore-single_type: {toxinidir}/centos/bluestore/single-type
centos-bluestore-single_type_dmcrypt: {toxinidir}/centos/bluestore/single-type-dmcrypt
centos-bluestore-mixed_type: {toxinidir}/centos/bluestore/mixed-type
centos-bluestore-mixed_type_dmcrypt: {toxinidir}/centos/bluestore/mixed-type-dmcrypt
centos-bluestore-mixed_type_explicit: {toxinidir}/centos/bluestore/mixed-type-explicit
centos-bluestore-mixed_type_dmcrypt_explicit: {toxinidir}/centos/bluestore/mixed-type-dmcrypt-explicit
commands=
git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch {env:CEPH_ANSIBLE_CLONE:"https://github.com/ceph/ceph-ansible.git"} {envdir}/tmp/ceph-ansible
python -m pip install -r {envdir}/tmp/ceph-ansible/tests/requirements.txt
ansible-galaxy install -r {envdir}/tmp/ceph-ansible/requirements.yml -v
ansible-galaxy collection install -r {envdir}/tmp/ceph-ansible/requirements.yml -v -p {envdir}/ansible_collections

# bash {toxinidir}/../scripts/vagrant_up.sh {env:VAGRANT_UP_FLAGS:""} {posargs:--provider=virtualbox}
bash {toxinidir}/../scripts/vagrant_up.sh {posargs:--provider=virtualbox}
Expand All @@ -44,9 +46,6 @@ commands=
# use ceph-ansible to deploy a ceph cluster on the vms
ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/deploy.yml --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} toxinidir={toxinidir}"

# prepare nodes for testing with testinfra
ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/setup.yml

# test cluster state using testinfra
py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests

Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
---

ceph_dev: True
cluster: test
public_network: "192.168.3.0/24"
cluster_network: "192.168.4.0/24"
monitor_interface: eth1
osd_objectstore: "bluestore"
osd_scenario: lvm
num_osds: 2
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
---

ceph_dev: True
cluster: test
public_network: "192.168.3.0/24"
cluster_network: "192.168.4.0/24"
monitor_interface: eth1
journal_size: 100
osd_objectstore: "bluestore"
osd_scenario: lvm
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,8 @@

dmcrypt: True
ceph_dev: True
cluster: test
public_network: "192.168.3.0/24"
cluster_network: "192.168.4.0/24"
monitor_interface: eth1
journal_size: 100
osd_objectstore: "bluestore"
osd_scenario: lvm
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
---

ceph_dev: True
cluster: test
public_network: "192.168.3.0/24"
cluster_network: "192.168.4.0/24"
monitor_interface: eth1
osd_objectstore: "bluestore"
osd_scenario: lvm
ceph_origin: 'repository'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,20 @@
become: yes
tasks:
- name: mark osds down
command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
command: "ceph osd down osd.{{ item }}"
with_items:
- 0
- 2

- name: destroy osd.2
command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
command: "ceph osd destroy osd.2 --yes-i-really-mean-it"
register: result
retries: 30
delay: 1
until: result is succeeded

- name: destroy osd.0
command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
command: "ceph osd destroy osd.0 --yes-i-really-mean-it"
register: result
retries: 30
delay: 1
Expand All @@ -41,7 +41,7 @@

# osd.2 device
- name: zap /dev/vdd1
command: "ceph-volume --cluster {{ cluster }} lvm zap /dev/vdd1 --destroy"
command: "ceph-volume lvm zap /dev/vdd1 --destroy"
environment:
CEPH_VOLUME_DEBUG: 1

Expand All @@ -57,18 +57,18 @@
state: present

- name: redeploy osd.2 using /dev/vdd1
command: "ceph-volume --cluster {{ cluster }} lvm create --bluestore --data /dev/vdd1 --osd-id 2"
command: "ceph-volume lvm create --bluestore --data /dev/vdd1 --osd-id 2"
environment:
CEPH_VOLUME_DEBUG: 1

# osd.0 lv
- name: zap test_group/data-lv1
command: "ceph-volume --cluster {{ cluster }} lvm zap test_group/data-lv1"
command: "ceph-volume lvm zap test_group/data-lv1"
environment:
CEPH_VOLUME_DEBUG: 1

- name: redeploy osd.0 using test_group/data-lv1
command: "ceph-volume --cluster {{ cluster }} lvm create --bluestore --data test_group/data-lv1 --osd-id 0"
command: "ceph-volume lvm create --bluestore --data test_group/data-lv1 --osd-id 0"
environment:
CEPH_VOLUME_DEBUG: 1

Expand All @@ -82,10 +82,10 @@
become: yes
tasks:
- name: mark osds down
command: "ceph --cluster {{ cluster }} osd down osd.0"
command: "ceph osd down osd.0"

- name: destroy osd.0
command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
command: "ceph osd destroy osd.0 --yes-i-really-mean-it"
register: result
retries: 30
delay: 1
Expand All @@ -98,12 +98,12 @@


- name: zap test_group/data-lv1
command: "ceph-volume --cluster {{ cluster }} lvm zap test_group/data-lv1"
command: "ceph-volume lvm zap test_group/data-lv1"
environment:
CEPH_VOLUME_DEBUG: 1

- name: prepare osd.0 using test_group/data-lv1
command: "ceph-volume --cluster {{ cluster }} lvm prepare --bluestore --data test_group/data-lv1 --osd-id 0"
command: "ceph-volume lvm prepare --bluestore --data test_group/data-lv1 --osd-id 0"
environment:
CEPH_VOLUME_DEBUG: 1

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,20 @@
become: yes
tasks:
- name: mark osds down
command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
command: "ceph osd down osd.{{ item }}"
with_items:
- 0
- 2

- name: destroy osd.2
command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
command: "ceph osd destroy osd.2 --yes-i-really-mean-it"
register: result
retries: 30
delay: 1
until: result is succeeded

- name: destroy osd.0
command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
command: "ceph osd destroy osd.0 --yes-i-really-mean-it"
register: result
retries: 30
delay: 1
Expand All @@ -44,7 +44,7 @@

# osd.2 device
- name: zap /dev/vdd1
command: "ceph-volume --cluster {{ cluster }} lvm zap /dev/vdd1 --destroy"
command: "ceph-volume lvm zap /dev/vdd1 --destroy"
environment:
CEPH_VOLUME_DEBUG: 1

Expand All @@ -60,18 +60,18 @@
state: present

- name: redeploy osd.2 using /dev/vdd1
command: "ceph-volume --cluster {{ cluster }} lvm create --bluestore --data /dev/vdd1 --osd-id 2"
command: "ceph-volume lvm create --bluestore --data /dev/vdd1 --osd-id 2"
environment:
CEPH_VOLUME_DEBUG: 1

# osd.0 device (zap without --destroy that removes the LV)
- name: zap test_group/data-lv1
command: "ceph-volume --cluster {{ cluster }} lvm zap test_group/data-lv1"
command: "ceph-volume lvm zap test_group/data-lv1"
environment:
CEPH_VOLUME_DEBUG: 1

- name: prepare osd.0 again using test_group/data-lv1
command: "ceph-volume --cluster {{ cluster }} lvm prepare --bluestore --data test_group/data-lv1 --osd-id 0"
command: "ceph-volume lvm prepare --bluestore --data test_group/data-lv1 --osd-id 0"
environment:
CEPH_VOLUME_DEBUG: 1

Expand Down Expand Up @@ -151,11 +151,11 @@

# zapping the first lv shouldn't remove the vg, allowing the second zap to succeed
- name: zap test_zap/data-lv1
command: "ceph-volume --cluster {{ cluster }} lvm zap --destroy test_zap/data-lv1"
command: "ceph-volume lvm zap --destroy test_zap/data-lv1"
environment:
CEPH_VOLUME_DEBUG: 1

- name: zap test_zap/data-lv2
command: "ceph-volume --cluster {{ cluster }} lvm zap --destroy test_zap/data-lv2"
command: "ceph-volume lvm zap --destroy test_zap/data-lv2"
environment:
CEPH_VOLUME_DEBUG: 1
19 changes: 9 additions & 10 deletions src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[tox]
envlist = centos8-bluestore-{create,prepare_activate,dmcrypt}
envlist = centos-bluestore-{create,prepare_activate,dmcrypt}
skipsdist = True

[testenv]
Expand All @@ -18,18 +18,20 @@ setenv=
VAGRANT_CWD = {changedir}
CEPH_VOLUME_DEBUG = 1
DEBIAN_FRONTEND=noninteractive
ANSIBLE_COLLECTIONS_PATH = {envdir}/ansible_collections
CEPH_ANSIBLE_VAGRANT_BOX = centos/stream8
changedir=
# plain/unencrypted
centos8-bluestore-create: {toxinidir}/centos8/bluestore/create
centos-bluestore-create: {toxinidir}/centos/bluestore/create
# dmcrypt
centos8-bluestore-dmcrypt: {toxinidir}/centos8/bluestore/dmcrypt
centos-bluestore-dmcrypt: {toxinidir}/centos/bluestore/dmcrypt
# TODO: these are placeholders for now, eventually we want to
# test the prepare/activate workflow of ceph-volume as well
centos8-bluestore-prepare_activate: {toxinidir}/xenial/bluestore/prepare_activate
centos-bluestore-prepare_activate: {toxinidir}/xenial/bluestore/prepare_activate
commands=
git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch {env:CEPH_ANSIBLE_CLONE:"https://github.com/ceph/ceph-ansible.git"} {envdir}/tmp/ceph-ansible
git clone -b {env:CEPH_ANSIBLE_BRANCH:main} --single-branch {env:CEPH_ANSIBLE_CLONE:"https://github.com/ceph/ceph-ansible.git"} {envdir}/tmp/ceph-ansible
pip install -r {envdir}/tmp/ceph-ansible/tests/requirements.txt
ansible-galaxy install -r {envdir}/tmp/ceph-ansible/requirements.yml -v
ansible-galaxy collection install -r {envdir}/tmp/ceph-ansible/requirements.yml -v -p {envdir}/ansible_collections

bash {toxinidir}/../scripts/vagrant_up.sh {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox}
bash {toxinidir}/../scripts/generate_ssh_config.sh {changedir}
Expand All @@ -45,10 +47,7 @@ commands=
cp {toxinidir}/../playbooks/deploy.yml {envdir}/tmp/ceph-ansible

# use ceph-ansible to deploy a ceph cluster on the vms
ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/deploy.yml --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} toxinidir={toxinidir}"

# prepare nodes for testing with testinfra
ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/setup.yml
ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/deploy.yml --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:main} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} toxinidir={toxinidir}"

# test cluster state using testinfra
py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests
Expand Down
Loading

0 comments on commit 63fe392

Please sign in to comment.