feat: support for Robot servers (#561)

Based on the Fork by Syself[0] and the Design Doc[1].

[0] https://github.com/syself/hetzner-cloud-controller-manager
[1] #523 (comment)

This ports most features of the fork while refactoring them to match
our coding style and the improvements I made in preparation for this.

Closes #525 #526 #527

---------

Co-authored-by: janiskemper <janis.kemper@syself.com>
Co-authored-by: Mawe Sprenger <mawe.sprenger@denkweit.de>
Co-authored-by: Thomas Guettler <thomas.guettler@syself.com>
Co-authored-by: Anurag <contact.anurag7@gmail.com>
Co-authored-by: batistein <sven.batista-steinbach@syself.com>

.github/workflows/test_e2e.yml

@@ -1,11 +1,11 @@
-name: Run e2e tests
+name: e2e tests
 on:
   pull_request: {}
   push:
     branches: [main]
 jobs:
-  k3s:
-    name: k3s ${{ matrix.k3s }}
+  cloud:
+    name: Cloud ${{ matrix.k3s }}
     permissions:
       id-token: write
     runs-on: ubuntu-latest
@@ -51,6 +51,7 @@ jobs:
           # make exported env variables available to following jobs
           echo "KUBECONFIG=$KUBECONFIG" >> "$GITHUB_ENV"
           echo "SKAFFOLD_DEFAULT_REPO=$SKAFFOLD_DEFAULT_REPO" >> "$GITHUB_ENV"
+          echo "CONTROL_IP=$CONTROL_IP" >> "$GITHUB_ENV"
 
       - name: Build and Deploy HCCM
         run: |
@@ -102,3 +103,127 @@ jobs:
         with:
           name: debug-logs-${{ env.SCOPE }}
           path: debug-logs/
+
+  robot:
+    name: Robot
+    permissions:
+      id-token: write
+
+    # Make sure that only one Job is using the server at a time
+    concurrency: robot-test-server
+    environment: e2e-robot
+
+    env:
+      K3S_CHANNEL: v1.28
+      SCOPE: gha-${{ github.run_id }}-${{ github.run_attempt }}-robot
+
+      # Disable routes in dev-env, not supported for Robot.
+      ROUTES_ENABLED: "false"
+      ROBOT_ENABLED: "true"
+      SERVER_NUMBER: ${{ vars.SERVER_NUMBER }}
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.21"
+      - uses: actions/checkout@master
+      - uses: hetznercloud/tps-action@main
+        with:
+          token: ${{ secrets.HCLOUD_TOKEN  }}
+      - uses: 3bit/setup-hcloud@v2
+      - uses: yokawasa/action-setup-kube-tools@v0.9.3
+        with:
+          setup-tools: |
+            helm
+            kubectl
+            skaffold
+          helm: v3.11.2
+          kubectl: v1.28.1
+          skaffold: v2.3.0
+      - name: Install k3sup
+        run: |
+          curl -sLS https://get.k3sup.dev | sh
+
+      - name: Setup test environment
+        env:
+          ROBOT_USER: ${{ secrets.ROBOT_USER }}
+          ROBOT_PASSWORD: ${{ secrets.ROBOT_PASSWORD }}
+        run: |
+          source <(hack/dev-up.sh)
+          
+          # make exported env variables available to following jobs
+          echo "KUBECONFIG=$KUBECONFIG" >> "$GITHUB_ENV"
+          echo "SKAFFOLD_DEFAULT_REPO=$SKAFFOLD_DEFAULT_REPO" >> "$GITHUB_ENV"
+          echo "CONTROL_IP=$CONTROL_IP" >> "$GITHUB_ENV"
+
+      - name: Build and Deploy HCCM
+        run: |
+          skaffold build --tag="e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}"
+          tag=$(skaffold build --tag="e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}" --quiet --output="{{ (index .Builds 0).Tag }}")
+          skaffold deploy \
+            --profile=robot \
+            --images=hetznercloud/hcloud-cloud-controller-manager=$tag
+
+      - name: Setup Robot Server
+        env:
+          ROBOT_USER: ${{ secrets.ROBOT_USER }}
+          ROBOT_PASSWORD: ${{ secrets.ROBOT_PASSWORD }}
+
+          # Nicer output
+          PY_COLORS: true
+          ANSIBLE_FORCE_COLOR: true
+        working-directory: hack/robot-e2e
+        run: |
+          ansible-galaxy install -r requirements.yml
+          echo "::group::ansible-playbook e2e-setup-robot-server.yml"
+          ansible-playbook e2e-setup-robot-server.yml -e scope=$SCOPE -e server_number=$SERVER_NUMBER -vvv
+          echo "::endgroup::"
+
+      - name: Run tests
+        env:
+          ROBOT_USER: ${{ secrets.ROBOT_USER }}
+          ROBOT_PASSWORD: ${{ secrets.ROBOT_PASSWORD }}
+        run: |
+          go test ./tests/e2e -tags e2e,robot -v -timeout 60m
+
+      - name: Download logs & events
+        if: always()
+        continue-on-error: true
+        run: |
+          mkdir debug-logs
+          kubectl logs \
+            --namespace kube-system \
+            --selector app.kubernetes.io/name=hcloud-cloud-controller-manager \
+            --all-containers \
+            --prefix=true \
+            --tail=-1 \
+            > debug-logs/hccm.log
+          
+          kubectl get events \
+            --all-namespaces \
+            --sort-by=.firstTimestamp \
+            --output yaml \
+          > debug-logs/events.yaml
+
+      - name: Show HCCM Logs on Failure
+        if: failure()
+        continue-on-error: true
+        run: |
+          echo "::group::hccm.log"
+          cat debug-logs/hccm.log
+          echo "::endgroup::"
+
+      - name: Cleanup test environment
+        if: always()
+        continue-on-error: true
+        run: |
+          hack/dev-down.sh
+
+      - name: Persist debug artifacts
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@v3
+        with:
+          name: debug-logs-${{ env.SCOPE }}
+          path: debug-logs/

.golangci.yaml

@@ -19,6 +19,11 @@ linters-settings:
       - pkg: k8s.io/apimachinery/pkg/apis/meta/v1
         alias: metav1
 
+      - pkg: github.com/syself/hrobot-go
+        alias: hrobot
+      - pkg: github.com/syself/hrobot-go/models
+        alias: hrobotmodels
+
   misspell:
     locale: "US"
 
@@ -58,3 +63,4 @@ issues:
     - path: internal/mocks
       linters:
         - unparam
+        - revive

README.md

@@ -254,6 +254,15 @@ alias kgp="kubectl get pods"
 alias kgs="kubectl get services"
 ```
 
+The test suite is split in three parts:
+
+- **General Part**: Sets up the test env & checks if the HCCM Pod is properly running
+   - Build Tag: `e2e`
+- **Cloud Part**: Tests regular functionality against a Cloud-only environment
+   - Build Tag: `e2e && !robot`
+- **Robot Part**: Tests Robot functionality against a Cloud+Robot environment
+   - Build Tag: `e2e && robot`
+
 ## Local test setup
 This repository provides [skaffold](https://skaffold.dev/) to easily deploy / debug this controller on demand
 

chart/templates/daemonset.yaml

@@ -73,6 +73,10 @@ spec:
             - name: HCLOUD_METRICS_ENABLED
               value: "false"
             {{- end }}
+            {{- if $.Values.robot.enabled }}
+            - name: ROBOT_ENABLED
+              value: "true"
+            {{- end }}
           image: {{ $.Values.image.repository }}:{{ tpl $.Values.image.tag . }} # x-release-please-version
           ports:
             {{- if $.Values.monitoring.enabled }}

chart/templates/deployment.yaml

@@ -77,6 +77,10 @@ spec:
             - name: HCLOUD_METRICS_ENABLED
               value: "false"
             {{- end }}
+            {{- if $.Values.robot.enabled }}
+            - name: ROBOT_ENABLED
+              value: "true"
+            {{- end }}
           image: {{ $.Values.image.repository }}:{{ tpl $.Values.image.tag . }} # x-release-please-version
           ports:
             {{- if $.Values.monitoring.enabled }}

chart/values.yaml

@@ -24,12 +24,27 @@ env:
   # The following two variables should *not* be set here:
   # HCLOUD_METRICS_ENABLED - see monitoring.enabled
   # HCLOUD_NETWORK - see networking.enabled
+  # ROBOT_ENABLED - see robot.enabled
+
   HCLOUD_TOKEN:
     valueFrom:
       secretKeyRef:
         name: hcloud
         key: token
 
+  ROBOT_USER:
+    valueFrom:
+      secretKeyRef:
+        name: hcloud
+        key: robot-user
+        optional: true
+  ROBOT_PASSWORD:
+    valueFrom:
+      secretKeyRef:
+        name: hcloud
+        key: robot-password
+        optional: true
+
 image:
   repository: hetznercloud/hcloud-cloud-controller-manager
   tag: '{{ $.Chart.Version }}'
@@ -81,3 +96,7 @@ additionalTolerations: []
 
 nodeSelector: {}
   # node-role.kubernetes.io/control-plane: ""
+
+robot:
+  # Set to true to enable support for Robot (Dedicated) servers.
+  enabled: false

deploy/ccm-networks.yaml

@@ -75,6 +75,18 @@ spec:
                 secretKeyRef:
                   key: token
                   name: hcloud
+            - name: ROBOT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  key: robot-password
+                  name: hcloud
+                  optional: true
+            - name: ROBOT_USER
+              valueFrom:
+                secretKeyRef:
+                  key: robot-user
+                  name: hcloud
+                  optional: true
             - name: HCLOUD_NETWORK
               valueFrom:
                 secretKeyRef:

deploy/ccm.yaml

@@ -72,6 +72,18 @@ spec:
                 secretKeyRef:
                   key: token
                   name: hcloud
+            - name: ROBOT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  key: robot-password
+                  name: hcloud
+                  optional: true
+            - name: ROBOT_USER
+              valueFrom:
+                secretKeyRef:
+                  key: robot-user
+                  name: hcloud
+                  optional: true
           image: hetznercloud/hcloud-cloud-controller-manager:v1.18.0 # x-release-please-version
           ports:
             - name: metrics

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/prometheus/client_golang v1.17.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4
+	github.com/syself/hrobot-go v0.2.5
 	k8s.io/api v0.28.4
 	k8s.io/apimachinery v0.28.4
 	k8s.io/client-go v0.28.4

go.sum

@@ -108,10 +108,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
-github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
@@ -301,6 +298,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/syself/hrobot-go v0.2.5 h1:Zs7GDFRd6fDn4YHYE9e5CGtRm6KYmMZwMMnm7OC/09g=
+github.com/syself/hrobot-go v0.2.5/go.mod h1:Oy47yZs+fJKcSh38S3OiNJdY34MXb0pkk796UnpYBnc=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
@@ -667,44 +666,24 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM=
-k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc=
 k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY=
 k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0=
-k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A=
-k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8=
 k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8=
 k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg=
-k8s.io/apiserver v0.28.3 h1:8Ov47O1cMyeDzTXz0rwcfIIGAP/dP7L8rWbEljRcg5w=
-k8s.io/apiserver v0.28.3/go.mod h1:YIpM+9wngNAv8Ctt0rHG4vQuX/I5rvkEMtZtsxW2rNM=
 k8s.io/apiserver v0.28.4 h1:BJXlaQbAU/RXYX2lRz+E1oPe3G3TKlozMMCZWu5GMgg=
 k8s.io/apiserver v0.28.4/go.mod h1:Idq71oXugKZoVGUUL2wgBCTHbUR+FYTWa4rq9j4n23w=
-k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4=
-k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo=
 k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY=
 k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4=
-k8s.io/cloud-provider v0.28.3 h1:9u+JjA3zIn0nqLOOa8tWnprFkffguSAhfBvo8p7LhBQ=
-k8s.io/cloud-provider v0.28.3/go.mod h1:shAJxdrKu+SwwGUhkodxByPjaH8KBFZqXo6jU1F0ehI=
 k8s.io/cloud-provider v0.28.4 h1:7obmeuJJ5CYTO9HANDqemf/d2v95U+F0t8aeH4jNOsQ=
 k8s.io/cloud-provider v0.28.4/go.mod h1:xbhmGZ7wRHgXFP3SNsvdmFRO87KJIvirDYQA5ydMgGA=
-k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI=
-k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8=
 k8s.io/component-base v0.28.4 h1:c/iQLWPdUgI90O+T9TeECg8o7N3YJTiuz2sKxILYcYo=
 k8s.io/component-base v0.28.4/go.mod h1:m9hR0uvqXDybiGL2nf/3Lf0MerAfQXzkfWhUY58JUbU=
-k8s.io/component-helpers v0.28.3 h1:te9ieTGzcztVktUs92X53P6BamAoP73MK0qQP0WmDqc=
-k8s.io/component-helpers v0.28.3/go.mod h1:oJR7I9ist5UAQ3y/CTdbw6CXxdMZ1Lw2Ua/EZEwnVLs=
 k8s.io/component-helpers v0.28.4 h1:+X9VXT5+jUsRdC26JyMZ8Fjfln7mSjgumafocE509C4=
 k8s.io/component-helpers v0.28.4/go.mod h1:8LzMalOQ0K10tkBJWBWq8h0HTI9HDPx4WT3QvTFn9Ro=
-k8s.io/controller-manager v0.28.3 h1:2s0wBvrGuRwMYEnl5Ed+qkK1kAfZR6H+0Ut1R2tHLRg=
-k8s.io/controller-manager v0.28.3/go.mod h1:lYu5hxBVmfK5NrpmeVrioPH4ROnE4OxmUM3xx6JWlLs=
 k8s.io/controller-manager v0.28.4 h1:8uJmo1pD6fWYk4mC/JfZQU6zPvuCgEHf3pd5G39ldDU=
 k8s.io/controller-manager v0.28.4/go.mod h1:pnO+UK2mcWNu1MxucqI8xHPD/8UBm04IUmp2u/3vbnM=
-k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
-k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
 k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
-k8s.io/kms v0.28.3 h1:jYwwAe96XELNjYWv1G4kNzizcFoZ50OOElvPansbw70=
-k8s.io/kms v0.28.3/go.mod h1:kSMjU2tg7vjqqoWVVCcmPmNZ/CofPsoTbSxAipCvZuE=
 k8s.io/kms v0.28.4 h1:PMgY/3CQTWP9eIKmNQiTgjLIZ0ns6O+voagzD2/4mSg=
 k8s.io/kms v0.28.4/go.mod h1:HL4/lR/bhjAJPbqycKtfhWiKh1Sp21cpHOL8P4oo87w=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=

hack/dev-up.sh

@@ -164,16 +164,28 @@ if [[ -n "${DEBUG:-}" ]]; then set -x; fi
     # Create HCLOUD_TOKEN Secret for hcloud-cloud-controller-manager.
     ( trap error ERR
       if ! kubectl -n kube-system get secret hcloud >/dev/null 2>&1; then
-        kubectl -n kube-system create secret generic hcloud --from-literal="token=$HCLOUD_TOKEN" --from-literal="network=$scope_name"
+        data=(
+          --from-literal="token=$HCLOUD_TOKEN"
+          --from-literal="network=$scope_name"
+        )
+        if [[ -v ROBOT_USER ]]; then
+          data+=(
+            --from-literal="robot-user=$ROBOT_USER"
+            --from-literal="robot-password=$ROBOT_PASSWORD"
+          )
+        fi
+        kubectl -n kube-system create secret generic hcloud "${data[@]}"
       fi) &
     wait
   ) &
   wait
   echo "Success - cluster fully initialized and ready, why not see for yourself?"
   echo '$ kubectl get nodes'
   kubectl get nodes
+  export CONTROL_IP=$(hcloud server ip "$scope_name-1")
 } >&2
 
 echo "export KUBECONFIG=$KUBECONFIG"
 $SCRIPT_DIR/registry-port-forward.sh
 echo "export SKAFFOLD_DEFAULT_REPO=localhost:30666"
+echo "export CONTROL_IP=$CONTROL_IP"

hack/robot-e2e/ansible.cfg

@@ -0,0 +1,7 @@
+[defaults]
+inventory = ${PWD}/inventory.yml
+host_key_checking = False
+stdout_callback = community.general.yaml
+
+[ssh_connection]
+pipelining = True