-
Notifications
You must be signed in to change notification settings - Fork 107
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This commit adds setup code to initialize & use Robot servers in e2e tests. It uses installimage with autosetup to prepare the server and then uses k3sup similar to our existing setup to join the server into the existing kubernetes cluster. Right now it only supports joining Robot servers as worker nodes, not for the control-plane. Also, it only supports non-Private Network setups.
- Loading branch information
Showing
11 changed files
with
353 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| [defaults] | ||
| inventory = ${PWD}/inventory.yml | ||
| host_key_checking = False | ||
|
|
||
| [ssh_connection] | ||
| pipelining = True |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| HOSTNAME {{ server_name }} | ||
|
|
||
| DRIVE1 /dev/sda | ||
| PART / ext4 all | ||
|
|
||
| IMAGE /root/.oldroot/nfs/images/Ubuntu-2204-jammy-amd64-base.tar.gz |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| --- | ||
| - name: Prepare Reinstall | ||
| hosts: localhost | ||
| connection: local | ||
| gather_facts: false | ||
|
|
||
| vars: | ||
| scope: dev | ||
| # Additional SSH keys to add to the server for debugging. Must already exist in Robot. | ||
| authorized_keys: [] | ||
|
|
||
| module_defaults: | ||
| group/community.hrobot.robot: | ||
| hetzner_user: "{{ lookup('ansible.builtin.env', 'ROBOT_USER') }}" | ||
| hetzner_password: "{{ lookup('ansible.builtin.env', 'ROBOT_PASSWORD') }}" | ||
|
|
||
| tasks: | ||
| - name: Get Server Info | ||
| community.hrobot.server_info: | ||
| server_number: "{{ server_number }}" | ||
| register: server_info | ||
|
|
||
| - name: Set Server Facts | ||
| ansible.builtin.set_fact: | ||
| server_ip: "{{ server_info.servers[0].server_ip }}" | ||
| server_name: "{{ server_info.servers[0].server_name }}" | ||
|
|
||
| - name: Create SSH Key | ||
| community.hrobot.ssh_key: | ||
| name: "hccm-{{ scope }}" | ||
| public_key: "{{ lookup('file', '../.ssh-{{ scope }}.pub') }}" | ||
| state: present | ||
| register: ssh_key | ||
|
|
||
| - name: Enable Rescue System | ||
| community.hrobot.boot: | ||
| server_number: "{{ server_number }}" | ||
| rescue: | ||
| authorized_keys: "{{ authorized_keys + [ ssh_key.fingerprint ] }}" | ||
| os: linux | ||
|
|
||
| - name: Reset Server (to get to Rescue System) | ||
| community.hrobot.reset: | ||
| server_number: "{{ server_number }}" | ||
| reset_type: hardware # only type that does not require a separate reset for starting again | ||
|
|
||
| - name: Wait for SSH | ||
| ansible.builtin.wait_for: | ||
| host: "{{ server_ip }}" | ||
| port: "{{ 22 }}" | ||
| search_regex: SSH | ||
|
|
||
| - name: Install OS to Server | ||
| hosts: all | ||
| gather_facts: false | ||
| tasks: | ||
| - name: Write autosetup | ||
| ansible.builtin.template: | ||
| src: autosetup.j2 | ||
| dest: /autosetup | ||
| vars: | ||
| server_name: "{{ hostvars['localhost']['server_name'] }}" | ||
|
|
||
| - name: installimage | ||
| # -t => Take over rescue system SSH public keys | ||
| ansible.builtin.command: /root/.oldroot/nfs/install/installimage -t yes | ||
|
|
||
| - name: Reboot | ||
| ansible.builtin.reboot: | ||
|
|
||
| - name: Create k3s directory | ||
| ansible.builtin.file: | ||
| path: /etc/rancher/k3s | ||
| state: directory | ||
|
|
||
| - name: Prepare Local Registry | ||
| ansible.builtin.copy: | ||
| src: ../k3s-registries.yaml | ||
| dest: /etc/rancher/k3s/registries.yaml | ||
|
|
||
| - name: Join Kubernetes Cluster | ||
| hosts: localhost | ||
| connection: local | ||
| gather_facts: false | ||
| vars: | ||
| control_ip: "{{ lookup('ansible.builtin.env', 'CONTROL_IP') }}" | ||
| k3s_channel: stable | ||
| scope: dev | ||
|
|
||
| tasks: | ||
| - name: k3sup | ||
| ansible.builtin.command: >- | ||
| k3sup join | ||
| --server-ip={{ control_ip | ansible.builtin.mandatory }} | ||
| --ip={{ server_ip }} | ||
| --k3s-channel={{ k3s_channel }} | ||
| --k3s-extra-args="--kubelet-arg cloud-provider=external --node-label instance.hetzner.cloud/is-root-server=true" | ||
| --ssh-key ../.ssh-{{ scope }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| all: | ||
| hosts: | ||
| # TODO: Dynamic inventory | ||
| hccm-test: | ||
| ansible_host: 142.132.203.104 | ||
| ansible_user: root | ||
| ansible_ssh_private_key_file: ../.ssh-{{ scope }} | ||
| ansible_ssh_common_args: '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| collections: | ||
| - name: community.hrobot |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| //go:build e2e | ||
| //go:build e2e && !robot | ||
|
|
||
| package e2e | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| //go:build e2e && robot | ||
|
|
||
| package e2e | ||
|
|
||
| import ( | ||
| "context" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
|
|
||
| "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/annotation" | ||
| "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/providerid" | ||
| ) | ||
|
|
||
| func TestRobotClientIsAvailable(t *testing.T) { | ||
| assert.NotNil(t, testCluster.hrobot) | ||
| } | ||
|
|
||
| func TestNodeSetCorrectNodeLabelsAndIPAddressesRobot(t *testing.T) { | ||
| t.Parallel() | ||
| ctx := context.Background() | ||
|
|
||
| // Get a random Robot server from all Nodes in the cluster | ||
| nodes, err := testCluster.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{ | ||
| LabelSelector: "instance.hetzner.cloud/is-root-server=true", | ||
| }) | ||
| assert.NoError(t, err) | ||
| assert.GreaterOrEqual(t, len(nodes.Items), 1) | ||
| node := nodes.Items[0] | ||
|
|
||
| // Parse the server number from the ProviderID | ||
| id, isCloudServer, err := providerid.ToServerID(node.Spec.ProviderID) | ||
| assert.NoError(t, err) | ||
| assert.False(t, isCloudServer) | ||
|
|
||
| // Get the server from the Robot API to cross-check Labels | ||
| server, err := testCluster.hrobot.ServerGet(int(id)) | ||
| assert.NoError(t, err) | ||
|
|
||
| labels := node.Labels | ||
| expectedLabels := map[string]string{ | ||
| "kubernetes.io/hostname": server.Name, | ||
| "kubernetes.io/os": "linux", | ||
| "kubernetes.io/arch": "amd64", | ||
| } | ||
| for expectedLabel, expectedValue := range expectedLabels { | ||
| assert.Equal(t, expectedValue, labels[expectedLabel], "node does not have expected label %s", expectedLabel) | ||
| } | ||
|
|
||
| expectedLabelsSet := []string{ | ||
| "node.kubernetes.io/instance-type", | ||
| "topology.kubernetes.io/region", | ||
| "topology.kubernetes.io/zone", | ||
| } | ||
| for _, expectedLabel := range expectedLabelsSet { | ||
| _, ok := labels[expectedLabel] | ||
| assert.True(t, ok, "node is missing expected label %s", expectedLabel) | ||
| } | ||
|
|
||
| for _, address := range node.Status.Addresses { | ||
| if address.Type == corev1.NodeExternalIP { | ||
| expectedIP := server.ServerIP | ||
| assert.Equal(t, expectedIP, address.Address, "node has unexpected external ip") | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func TestServiceLoadBalancersRobot(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| lbTest := lbTestHelper{ | ||
| t: t, | ||
| K8sClient: testCluster.k8sClient, | ||
| podName: "loadbalancer-robot-only", | ||
| } | ||
|
|
||
| pod := lbTest.DeployTestPod() | ||
|
|
||
| lbSvc := lbTest.ServiceDefinition(pod, map[string]string{ | ||
| string(annotation.LBLocation): "nbg1", | ||
| // Only add the Robot server as a Load Balancer target | ||
| string(annotation.LBNodeSelector): "instance.hetzner.cloud/is-root-server=true", | ||
| }) | ||
|
|
||
| lbSvc, err := lbTest.CreateService(lbSvc) | ||
| assert.NoError(t, err) | ||
|
|
||
| WaitForHTTPAvailable(t, lbSvc.Status.LoadBalancer.Ingress[0].IP, false) | ||
|
|
||
| lbTest.TearDown() | ||
| } |
Oops, something went wrong.