Add a workflow for external tests to ec2 (#126)

- Create external ec2 nodes in a daily cron (issue #114).
- Matrixes spin up Flannel and Patu in parallel.
- Run pod to pod iperf and displays the results.

Signed-off-by: Brent Salisbury <bsalisbu@redhat.com>

.github/workflows/periodic.yaml

@@ -0,0 +1,55 @@
+name: Patu Periodic Cloud Performance and Scale Testing
+
+on:
+  schedule:
+    - cron: '0 12 * * *'
+
+jobs:
+  deploy-perf-scale:
+    name: deploy-perf-scale
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        cni: ["patu", "flannel"]
+        kube-distribution: ["kubeadm"]
+    env:
+      JOB_NAME: "patu-periodic-perfscale-${{ matrix.k8s-distro }}-${{ matrix.cni }}"
+      MATRIX_CNI: ${{ matrix.cni }}
+      KUBE_DIST: ${{ matrix.k8s-distro }}
+      AWS_REGION: "us-east-1"
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+    - name: checkout
+      uses: actions/checkout@v2
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: Install Ansible and dependencies
+      run: pip3.10 install boto boto3 ansible-vault ansible-core==2.13.3
+
+    - name: Install amazon.aws Ansible library
+      run: ansible-galaxy collection install amazon.aws
+
+    - name: Create ansible ssh key
+      run: |
+        echo "${{ secrets.ANSIBLE_SSH_KEY }}" > ./test/ansible/periodic/patu-ci.pem
+        chmod 0400 ./test/ansible/periodic/patu-ci.pem
+
+    - name: Create vault password file
+      run: |
+        echo "${{ secrets.ANSIBLE_VAULT_PASSWORD }}" > /home/runner/work/patu/patu/vault-secret.txt
+        chmod 0400 vault-secret.txt
+
+    - name: Deploy EC2 Playbooks
+      run: |
+        ansible-playbook -vv ./test/ansible/periodic/deploy.yml --extra-vars "MATRIX_CNI=${{ matrix.cni }}" --vault-password-file /home/runner/work/patu/patu/vault-secret.txt
+        rm vault-secret.txt
+        rm patu-ci.pem
+
+    - name: Display Iperf3 Results for ${{ matrix.cni }}
+      run: cat ./test/ansible/periodic/iperf-results-${{ matrix.cni }}.txt

.gitignore

@@ -58,3 +58,6 @@ dkms.conf
 
 # jetbrains config files
 .idea/
+
+# miscellaneous
+*.pem

.licenserc.yaml

@@ -34,3 +34,4 @@ header:
     - '**/*.yaml'
     - '**/*.yml'
     - '.clang-format'
+    - 'test/ansible/'

test/ansible/periodic/ansible.cfg

@@ -0,0 +1,10 @@
+[defaults]
+host_key_checking = false
+deprecation_warnings = false
+ask_pass = false
+stdout_callback = yaml
+remote_user = ubuntu
+# defaults to the base directory in the project
+inventory = inventory.txt
+# create .pem private_key_file and provide location
+private_key_file = patu-ci.pem

test/ansible/periodic/deploy.yml

@@ -0,0 +1,64 @@
+# roles get branched from here
+- hosts: localhost
+  vars_files:
+    - vars.yml
+  roles:
+    - role: setup-ec2
+
+- hosts: singleNodeCluster
+  roles:
+    - role: install-kubeadm
+      environment:
+        KUBECONFIG: /home/{{ ansible_user }}/.kube/config
+      when: MATRIX_CNI == "patu"
+
+- hosts: singleNodeCluster
+  roles:
+    - role: install-kubeadm
+      environment:
+        KUBECONFIG: /home/{{ ansible_user }}/.kube/config
+      when: MATRIX_CNI == "flannel"
+
+- hosts: singleNodeCluster
+  roles:
+    - role: install-cni
+      environment:
+        KUBECONFIG: /home/{{ ansible_user }}/.kube/config
+      when: MATRIX_CNI == "patu"
+
+- hosts: singleNodeCluster
+  roles:
+    - role: install-cni
+      environment:
+        KUBECONFIG: /home/{{ ansible_user }}/.kube/config
+      when: MATRIX_CNI == "flannel"
+
+- hosts: singleNodeCluster
+  roles:
+    - role: run-iperf
+      environment:
+        KUBECONFIG: /home/{{ ansible_user }}/.kube/config
+      vars:
+        MATRIX: patu-kpng-kubeadm
+      when: MATRIX_CNI == "patu"
+
+- hosts: singleNodeCluster
+  roles:
+    - role: run-iperf
+      environment:
+        KUBECONFIG: /home/{{ ansible_user }}/.kube/config
+      vars:
+        MATRIX: flannel-kubeproxy-kubeadm
+      when: MATRIX_CNI == "flannel"
+
+- hosts: singleNodeCluster
+  roles:
+    - role: reset-kubeadm
+
+# TODO: cleanup using explicit node names from inventory instead of NodeTag
+# TODO: but what about a scenario where the runners are spun up but a step fails?
+#- hosts: localhost
+#  vars_files:
+#    - vars.yml
+#  roles:
+#    - role: terminate-ec2

test/ansible/periodic/install-cni/tasks/main.yml

@@ -0,0 +1,47 @@
+---
+# tasks file for install-cni
+- name: Verify kubectl
+  command: kubectl get pods --all-namespaces
+
+### Patu Installer Section ###
+- name: Copy the Patu repo to the remote host
+  copy:
+    src: ../../../../patu/
+    dest: /home/{{ ansible_user }}/patu/
+  when: MATRIX_CNI == "patu"
+
+- name: Change file ownership, group and permissions
+  ansible.builtin.file:
+    path: "/home/{{ ansible_user }}/patu/deploy/kubernetes/patu-installer"
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: "0755"
+  when: MATRIX_CNI == "patu"
+
+- name: Install KPNG and Patu
+  shell: |
+    PATU_CONFIG=/home/{{ ansible_user }}/patu/deploy/patu.yaml \
+    KPNG_CONFIG=/home/{{ ansible_user }}/patu/deploy/kpngebpf.yaml \
+    /home/{{ ansible_user }}/patu/deploy/kubernetes/patu-installer apply all
+  when: MATRIX_CNI == "patu"
+
+- name: Wait for CoreDNS pods to become ready
+  shell: kubectl wait --for=condition=ready pods -l k8s-app=kube-dns -n kube-system --timeout=30s
+  when: MATRIX_CNI == "patu"
+
+### Flannel Installer Section ###
+- name: Deploy kubeadm for the Flannel CNI for the Flannel matrix
+  shell: kubectl apply -f https://github.com/coreos/flannel/raw/master/Documentation/kube-flannel.yml
+  when: MATRIX_CNI == "flannel"
+
+- name: Remove kubeadm taints
+  shell: kubectl taint nodes --all node-role.kubernetes.io/control-plane- node-role.kubernetes.io/master-
+  when: MATRIX_CNI == "flannel"
+
+- name: Pause for flannel convergence
+  pause:
+    seconds: 10
+  when: MATRIX_CNI == "flannel"
+
+- name: Display kube pods
+  command: kubectl get pods --all-namespaces

test/ansible/periodic/install-cni/vars/main.yml

@@ -0,0 +1,2 @@
+---
+# vars file for install-cni

test/ansible/periodic/install-kubeadm/tasks/main.yml

@@ -0,0 +1,108 @@
+---
+# tasks file for install-kubeadm
+
+- name: Update repo cache
+  become: yes
+  apt:
+    update_cache: yes
+
+- name: Install dependencies
+  become: yes
+  apt:
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - curl
+      - gnupg2
+      - software-properties-common
+    state: latest
+
+- name: Host configurations
+  shell: |
+    sudo sysctl -w net.ipv4.ip_forward=1
+    sudo modprobe br_netfilter
+
+- name: Host configurations
+  shell: |
+    sudo sysctl -w net.ipv4.ip_forward=1
+    sudo modprobe br_netfilter
+
+- name: Configure cri-o repos
+  vars:
+    OS: "xUbuntu_20.04"
+    CRIO_VERSION: "1.23"
+  shell: |
+    echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ OS }}/ /"|sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
+    echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ CRIO_VERSION }}/{{ OS }}/ /"|sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:{{ CRIO_VERSION }}.list
+    curl -L https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable:cri-o:{{ CRIO_VERSION }}/{{ OS }}/Release.key | sudo apt-key add -
+    curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ OS }}/Release.key | sudo apt-key add -
+  ignore_errors: true
+
+- name: Configure kube repos
+  shell: |
+    sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
+    echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
+
+- name: Update repo cache
+  become: yes
+  apt:
+    update_cache: yes
+
+- name: Install cri-o
+  become: yes
+  apt:
+    name:
+      - cri-o
+      - cri-o-runc
+    state: latest
+
+- name: Enable cri-o systemd
+  shell: |
+    sudo systemctl enable crio.service
+    sudo systemctl start crio.service
+
+- name: Install kube binaries
+  vars:
+    K8S_VERSION: "1.24.4-00"
+  shell: sudo apt install -y kubeadm={{ K8S_VERSION }} kubelet={{ K8S_VERSION }} kubectl={{ K8S_VERSION }}
+
+- name: Deploy kubeadm for the Patu matrix with kubeproxy disabled
+  shell: sudo kubeadm init --upload-certs --pod-network-cidr=10.200.0.0/16 --v=6 --skip-phases=addon/kube-proxy
+  when:  MATRIX_CNI == "patu"
+
+- name: Deploy kubeadm for the Flannel CNI matrix with kubeproxy enabled
+  shell: sudo kubeadm init --pod-network-cidr=10.244.0.0/16
+  when:  MATRIX_CNI == "flannel"
+
+- name: Wait for kubeconfig to be created
+  become: yes
+  wait_for:
+    path: /etc/kubernetes/admin.conf
+    state: present
+    timeout: 30
+  ignore_errors: True
+
+- name: Creating the .kube directory
+  file:
+    path: /home/{{ ansible_user }}/.kube/
+    state: directory
+
+- name: Copying kubeconfig to .kube directory
+  become: yes
+  copy:
+    remote_src: yes
+    src: /etc/kubernetes/admin.conf
+    dest: /home/{{ ansible_user }}/.kube/config
+
+- name: Change the owner of .kube/config
+  shell: "sudo chown $(id -u {{ ansible_user }}):$(id -g {{ ansible_user }}) /home/{{ ansible_user }}/.kube/config"
+
+- name: export KUBECONFIG
+  shell: export KUBECONFIG=/home/{{ ansible_user }}/.kube/config
+
+- name: Pause for convergence
+  pause:
+    seconds: 15
+
+- name: Verify kubectl
+  command: kubectl get pods --all-namespaces

test/ansible/periodic/install-kubeadm/vars/main.yml

@@ -0,0 +1,2 @@
+---
+# vars file for install-kubeadm

test/ansible/periodic/inventory.txt

@@ -0,0 +1 @@
+[singleNodeCluster]

test/ansible/periodic/reset-kubeadm/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+- name: Verify kubectl
+  command: kubectl get pods --all-namespaces
+
+- name: Reset kubeadm
+  shell: |
+    sudo kubeadm -f reset
+    sudo crictl rm -f `crictl ps -a | grep "k8s_" | awk '{print $1}'`
+    # Remove all the patu images.
+    sudo apt purge kubectl kubeadm kubelet kubernetes-cni -y --allow-change-held-packages && apt autoremove -y
+    sudo rm -fr /etc/kubernetes/; sudo rm -fr ~/.kube/; sudo rm -fr /var/lib/etcd; sudo rm -rf /var/lib/cni/
+    sudo systemctl restart crio.service
+    sudo systemctl daemon-reload
+    sudo iptables -F
+    sudo iptables -t nat -F
+    sudo iptables -t mangle -F
+    sudo iptables -X
+    sudo iptables -L

test/ansible/periodic/reset-kubeadm/vars/main.yml

@@ -0,0 +1,2 @@
+---
+# vars file for reset-kubeadm