WIP establish regular molecule tests

.github/workflows/molecule.yaml

@@ -0,0 +1,25 @@
+---
+name: Ansible Molecule Tests - Default
+
+# yamllint disable-line rule:truthy
+on:
+  push:
+    branches:
+      - main
+      - release-*
+    paths:
+      - 'ansible'
+  pull_request:
+    branches:
+      - main
+      - release-*
+jobs:
+  test:
+    runs-on: self-hosted
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Test
+        run: make devkit.run
+        env:
+          WHAT: make molecule_test_default

.github/workflows/molecule_gpu.yaml

@@ -0,0 +1,25 @@
+---
+name: Ansible Molecule Tests - Default
+
+# yamllint disable-line rule:truthy
+on:
+  push:
+    branches:
+      - main
+      - release-*
+    paths:
+      - 'ansible/roles/gpu'
+  pull_request:
+    branches:
+      - main
+      - release-*
+jobs:
+  test:
+    runs-on: self-hosted
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Test
+        run: make devkit.run
+        env:
+          WHAT: make molecule_test_gpu

Dockerfile

@@ -50,5 +50,10 @@ COPY images /root/images
 COPY ansible /root/ansible
 COPY packer /root/packer
 
+# this is needed for containerd tar
+# place it in /usr/share/ansible/collections, the container will be run with a different user
+RUN mkdir -p /usr/share/ansible/collections && ansible-galaxy collection install ansible.utils -p /usr/share/ansible/collections
+RUN mkdir -p /usr/share/ansible/collections && ansible-galaxy collection install community.general -p /usr/share/ansible/collections
+
 WORKDIR /root
 ENTRYPOINT ["/usr/local/bin/konvoy-image"]

Dockerfile.devkit

@@ -133,6 +133,12 @@ RUN if [ "${DOCKER_GID}" -ne "${GROUP_ID}" ]; then \
         rm -rf /tmp/docker.group*; \
     fi
 
+# this is needed for containerd tar
+# place it in /usr/share/ansible/collections, the container will be run with a different user
+RUN mkdir -p /usr/share/ansible/collections && ansible-galaxy collection install ansible.utils -p /usr/share/ansible/collections
+RUN mkdir -p /usr/share/ansible/collections && ansible-galaxy collection install community.general -p /usr/share/ansible/collections
+
+
 # hadolint ignore=DL3059
 RUN --mount=type=secret,id=githubtoken PACKER_GITHUB_API_TOKEN="$(cat /run/secrets/githubtoken)" export PACKER_GITHUB_API_TOKEN && \
     packer-${BUILDARCH} plugins install github.com/hashicorp/googlecompute ">=1.0.11" && \

Makefile

@@ -172,6 +172,7 @@ $(ENVSUBST_ASSETS)/envsubst:
 	chmod +x $(ENVSUBST_ASSETS)/envsubst
 
 
+include make/molecule.mk
 include hack/pip-packages/Makefile
 include test/infra/aws/Makefile
 include test/infra/vsphere/Makefile

ansible/molecule/default/molecule.yml

@@ -10,37 +10,56 @@ platforms:
     region: us-east-1
     instance_type: t3.small
     ssh_user: centos
+    spot_price: false
+  - name: konvoyimage-rhel7.9-${USER:-ci}-${HOSTNAME:-local}
+    image_search_name: "RHEL-7.9_HVM-*"
+    image_search_owner: "309956199498"
+    region: us-east-1
+    instance_type: t3.small
+    spot_price: false
+    ssh_user: ec2-user
+  - name: konvoyimage-rhel8.6-${USER:-ci}-${HOSTNAME:-local}
+    image_search_name: "RHEL-8.6.0_HVM-*"
+    image_search_owner: "309956199498"
+    region: us-east-1
+    instance_type: t3.small
+    spot_price: false
+    ssh_user: ec2-user
   - name: konvoyimage-flatcar-stable-${USER:-ci}-${HOSTNAME:-local}
     image_search_name: "Flatcar*stable*"
     image_search_owner: "075585003325"
     region: us-east-1
     instance_type: t3.small
-    spot_price: false
     ssh_user: core
+    spot_price: false
   - name: konvoyimage-sles15-sp3-${USER:-ci}-${HOSTNAME:-local}
     image_search_name: "suse-sles-15-sp3-v*"
     image_search_owner: "amazon"
     region: us-east-1
     instance_type: t3.small
     ssh_user: ec2-user
+    spot_price: false
   - name: konvoyimage-oracle-79-${USER:-ci}-${HOSTNAME:-local}
     image_search_name: "OL7.9-x86_64-HVM-2020-12-07"
     image_search_owner: "131827586825"
     region: us-east-1
     instance_type: t3.small
     ssh_user: ec2-user
+    spot_price: false
   - name: konvoyimage-ubuntu-18.04-${USER:-ci}-${HOSTNAME:-local}
     image_search_name: "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server*"
     image_search_owner: "099720109477"
     region: us-east-1
     instance_type: t3.small
     ssh_user: ubuntu
+    spot_price: false
   - name: konvoyimage-ubuntu-20.04-${USER:-ci}-${HOSTNAME:-local}
     image_search_name: "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server*"
     image_search_owner: "099720109477"
     region: us-east-1
     instance_type: t3.small
     ssh_user: ubuntu
+    spot_price: false
 
 provisioner:
   name: ansible

ansible/molecule/default/tests/test_cloudinit.py

@@ -0,0 +1,50 @@
+import os
+import pytest
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ["MOLECULE_INVENTORY_FILE"]
+).get_hosts("all")
+
+def test_cloudinit_feature_flags(host):
+    """
+    ubuntu 18.04: does not need the feature flag
+    all other except flatcar: expect feature overrides
+    """
+    distro = host.system_info.distribution
+    release = host.system_info.release
+
+    # for flatcar we can skip
+    if distro == "flatcar":
+        pytest.skip("no changes on flatcar")
+
+    # if cloud-init is lower than 20.0 we can skip
+    cloud_init_version = host.run("cloud-init --version")
+    assert cloud_init_version.succeeded
+
+    cloud_init_version_str = cloud_init_version.stdout.strip('\n')
+    if not cloud_init_version_str:
+        cloud_init_version_str = cloud_init_version.stderr.strip('\n')
+
+    assert cloud_init_version_str
+
+    cloud_init_version_str_version_part = cloud_init_version_str.split(" ")[-1]
+    major_version = cloud_init_version_str_version_part.split(".")[0]
+
+    if int(major_version) < 20:
+        pytest.skip("cloud-init major version ({}) below 20".format(major_version))
+
+    if distro != "ubuntu":
+        cmd = host.run("python3 -c \"import sysconfig; print(sysconfig.get_path('purelib'))\"")
+        assert cmd.succeeded
+
+        featurefile = host.file("{}/cloudinit/feature_overrides.py".format(cmd.stdout.strip('\n')))
+        assert featurefile.exists
+        assert b'ERROR_ON_USER_DATA_FAILURE = False' in featurefile.content
+    # ubuntu 18.04 still supported and no need for this feature flag
+    elif distro == "ubuntu" and not release == "18.04":
+        featurefile = host.file("/usr/lib/python3/dist-packages/cloudinit/feature_overrides.py")
+        assert featurefile.exists
+        assert b'ERROR_ON_USER_DATA_FAILURE = False' in featurefile.content
+    else:
+        assert True

ansible/molecule/default/tests/test_default.py

@@ -1,5 +1,4 @@
 import os
-
 import pytest
 import testinfra.utils.ansible_runner
 
@@ -78,3 +77,36 @@ def test_cloudinit_feature_flags(host):
         assert b"ERROR_ON_USER_DATA_FAILURE = False" in featurefile.content
     else:
         assert True
+
+def test_kubelet_kubectl_installed(host):
+    """
+    we expect kubectl and kubelet package to be installed
+
+    flatcar: skip no packages
+    """
+    distro = host.system_info.distribution
+    if distro == "flatcar":
+        pytest.skip("no packages on flatcar")
+
+    assert host.package("kubectl").is_installed
+    assert host.package("kubelet").is_installed
+
+def test_kubeadm_installed(host):
+    """
+    we expect kubeadm package to be installed
+
+    flatcar: skip no packages
+    """
+    distro = host.system_info.distribution
+    if distro == "flatcar":
+        pytest.skip("no packages on flatcar")
+
+    assert host.package("kubeadm").is_installed
+
+def test_kube_cmd_path(host):
+    """
+    kubelet, kubeadm and kubectl must be in path
+    """
+    assert host.exists("kubelet")
+    assert host.exists("kubeadm")
+    assert host.exists("kubectl")

ansible/molecule/default/tests/test_kubernetes.py

@@ -0,0 +1,56 @@
+import os
+import pytest
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ["MOLECULE_INVENTORY_FILE"]
+).get_hosts("all")
+
+def test_kubelet_kubectl_installed(host):
+    """
+    we expect kubectl and kubelet package to be installed
+
+    flatcar: skip no packages
+    """
+    distro = host.system_info.distribution
+    if distro == "flatcar":
+        pytest.skip("no packages on flatcar")
+
+    assert host.package("kubectl").is_installed
+    assert host.package("kubelet").is_installed
+
+def test_kubeadm_installed(host):
+    """
+    we expect kubeadm package to be installed
+
+    flatcar: skip no packages
+    """
+    distro = host.system_info.distribution
+    if distro == "flatcar":
+        pytest.skip("no packages on flatcar")
+
+    assert host.package("kubeadm").is_installed
+
+def test_kube_cmd_path(host):
+    """
+    kubelet, kubeadm and kubectl must be in path
+    """
+    distro = host.system_info.distribution
+    if distro == "flatcar":
+        pytest.skip("flatcar uses different PATH for non-interactive")
+
+    assert host.exists("kubelet")
+    assert host.exists("kubeadm")
+    assert host.exists("kubectl")
+
+def test_kube_cmd_path_flatcar(host):
+    distro = host.system_info.distribution
+    if distro != "flatcar":
+        pytest.skip("flatcar uses different PATH for non-interactive")
+    # the path is only set on interactive shell. SO lets append it here
+    cmd = host.run("bash -c 'PATH=$PATH:/opt/bin type kubectl'")
+    assert cmd.succeeded is True
+    cmd = host.run("bash -c 'PATH=$PATH:/opt/bin type kubelet'")
+    assert cmd.succeeded is True
+    cmd = host.run("bash -c 'PATH=$PATH:/opt/bin type kubeadm'")
+    assert cmd.succeeded is True

ansible/molecule/ec2/converge.yml

@@ -8,4 +8,4 @@
         sysprep: false
 
 - name: "Include Provision"
-  include: ../../provision.yaml
+  import_playbook: ../../provision.yaml

ansible/molecule/ec2/create.yml

@@ -70,33 +70,32 @@
         - item.image_search_name is defined and item.image_search_name != ""
       register: platform_with_images
 
-
-    - name: Create molecule instance(s)
-      ec2:
+    - name: Create molecule instances
+      ec2_instance:
+        name: "molecule-{{ item.item.name }}"
         key_name: "{{ keypair_name }}"
-        image: "{{ item.item.image if item.skipped is defined and item.skipped else (item.images | sort(attribute='creation_date') | last).image_id }}"
+        image_id: "{{ item.item.image if item.item.image is defined else (item.images | sort(attribute='creation_date') | last).image_id if item.images|length >0 else 'UNDEFINED' }}"
         region: "{{ item.item.region }}"
-        zone: "{{ item.item.zone|default(omit) }}"
+        availability_zone: "{{ item.item.zone|default(omit) }}"
         instance_initiated_shutdown_behavior: terminate
         instance_type: "{{ item.item.instance_type }}"
-        spot_price: "{{ item.item.spot_price | ternary(item.item.spot_price, omit) if item.item.spot_price is defined else spot_price_max_calc }}"
-        group: "{{ security_group_name }}"
+        security_group: "{{ security_group_name }}"
+        network:
+          assign_public_ip: true
         volumes:
           - device_name: "{{ item.item.volume_name | default('/dev/sda1') }}"
-            delete_on_termination: true
-            volume_size: "{{ item.item.volume_size | default(20) }}"
-            volume_type: gp2
-        instance_tags:
-          Name: "molecule {{ item.item.name }}"
+            ebs:
+              delete_on_termination: true
+              volume_size: "{{ item.item.volume_size | default(20) }}"
+              volume_type: gp2
+        tags:
           instance: "{{ item.item.name }}"
           molecule_region: "{{ item.item.region }}"
           molecule_ssh_user: "{{ item.item.ssh_user | default(ssh_user) }}"
           owner: ${USER:-ci}
           expires: 2h
         wait: yes
         exact_count: 1
-        count_tag:
-          instance: "{{ item.item.name }}"
       register: server
       loop: "{{ platform_with_images.results }}"
       loop_control:
@@ -114,19 +113,30 @@
 
     # Mandatory configuration for Molecule to function.
 
+    - name: wait for public ip
+      ec2_instance_info:
+        instance_ids:
+          - "{{ item.instances[0].instance_id }}"
+      with_items: "{{ ec2_jobs.results }}"
+      register: instances_info
+      retries: 5
+      delay: 10
+      until: (instances_info.instances[0].network_interfaces | first).attachment.status == "attached"
+      when: server.changed | bool
+
     - name: Populate instance config dict
       set_fact:
         instance_conf_dict:
           {
             "instance": "{{ item.instances[0].tags.instance }}",
             "region": "{{ item.instances[0].tags.molecule_region }}",
-            "address": "{{ item.instances[0].public_ip }}",
+            "address": "{{ (item.instances[0].network_interfaces | first).association.public_ip }}",
             "user": "{{ item.instances[0].tags.molecule_ssh_user }}",
             "port": "{{ ssh_port }}",
             "identity_file": "{{ keypair_path }}",
-            "instance_ids": "{{ item.instance_ids }}",
+            "instance_ids": "{{ item.instances | map(attribute='instance_id') | list }}",
           }
-      with_items: "{{ ec2_jobs.results }}"
+      with_items: "{{ instances_info.results }}"
       register: instance_config_dict
       when: server.changed | bool
 

ansible/molecule/ec2/destroy.yml

@@ -26,7 +26,7 @@
       when: not molecule_instance_config_stat.stat.exists
 
     - name: Destroy molecule instances
-      ec2:
+      ec2_instance:
         state: absent
         instance_ids: "{{ item.instance_ids }}"
         region: "{{ item.region }}"

ansible/roles/containerd/tasks/debian.yaml

@@ -5,6 +5,7 @@
   failed_when:
     - remove_version_hold.rc != 0
     - remove_version_hold.rc != 100  # package does not exist
+  changed_when: remove_version_hold.rc == 0
 
 # remove previous install with KIB through an OS package
 - name: remove containerd.io deb package