Adding Windows Goss support

kubernetes-sigs · Mar 29, 2021 · 4d8e4cd · 4d8e4cd
1 parent 88639f4
commit 4d8e4cd
Show file tree

Hide file tree

Showing 14 changed files with 324 additions and 10 deletions.
diff --git a/docs/book/src/capi/goss/goss.md b/docs/book/src/capi/goss/goss.md
@@ -14,8 +14,9 @@ to test if the images have all requisite components to work with cluster API.
 |----|---------|
 | Amazon Linux | aws
 | PhotonOS | ova
-| Ubuntu | aws , ova, azure
+| Ubuntu | aws, azure, ova
 | CentOS | aws, ova 
+| Windows | aws, azure, ova
 
 
 ### Prerequisites for Running GOSS
@@ -31,7 +32,7 @@ Supported arguments are passed through file: `packer/config/goss-args.json`
   "goss_url": "",
   "goss_format_options": "pretty",
   "goss_vars_file": "packer/goss/goss-vars.yaml",
-  "goss_version": "0.3.13"
+  "goss_version": "0.3.16"
 }
 ```
 ##### Supported values for some of the arguments can be found [here](https://github.com/aelsabbahy/goss).

diff --git a/images/capi/Makefile b/images/capi/Makefile
@@ -178,7 +178,8 @@ COMMON_WINDOWS_VAR_FILES :=	packer/config/kubernetes.json \
 					packer/config/windows/docker.json \
 					packer/config/windows/ansible-args-windows.json \
 					packer/config/windows/common.json \
-					packer/config/windows/cloudbase-init.json
+					packer/config/windows/cloudbase-init.json \
+					packer/config/windows/goss-args.json 
 
 # Initialize a list of flags to pass to Packer. This includes any existing flags
 # specified by PACKER_FLAGS, as well as prefixing the list with the variable

diff --git a/images/capi/hack/ensure-goss.sh b/images/capi/hack/ensure-goss.sh
@@ -23,9 +23,9 @@ set -o pipefail
 source hack/utils.sh
 
 # SHA are for amd64 arch.
-_version="2.0.0"
-darwin_sha256="be09a793cb63e898895e9d371eb9015ab2ca7c8b5e929c1d79bafc7e23e871e0"
-linux_sha256="97ed6de22ba8f1f7d9cefa6234e771121c2918563057c44a0615c47de98391e4"
+_version="3.0.2"
+darwin_sha256="4469555b6a78322791f8df2cb2461b9b594ad250033bfdbc28b93622d2042789"
+linux_sha256="15f63e7bb83f0c32cd8d823dac0d4e1fef2a513f701315d91d84b4197a39dcd8"
 _bin_url="https://github.com/YaleUniversity/packer-provisioner-goss/releases/download/v${_version}/packer-provisioner-goss-v${_version}-${HOSTOS}-${HOSTARCH}.tar.gz"
 _tarfile="${HOME}/.packer.d/plugins/packer-provisioner-goss.tar.gz"
 _binfile="${HOME}/.packer.d/plugins/packer-provisioner-goss"
@@ -48,7 +48,7 @@ esac
 if [ -f "${_binfile}" ]; then
   current_shasum=$(get_shasum "${_binfile}")
   if [ "$current_shasum" != "$_sha256" ]; then
-    echo "Wrong version of binary present."
+    echo "Wrong version of binary present. $current_shasum"
   else
     echo "Right version of binary present"
     # Check if binary is executable.

diff --git a/images/capi/hack/ensure-packer.sh b/images/capi/hack/ensure-packer.sh
@@ -20,7 +20,7 @@ set -o pipefail
 
 [[ -n ${DEBUG:-} ]] && set -o xtrace
 
-_version="1.6.6"
+_version="1.7.0"
 
 # Change directories to the parent directory of the one in which this
 # script is located.

diff --git a/images/capi/packer/ami/packer-windows.json b/images/capi/packer/ami/packer-windows.json
@@ -94,6 +94,36 @@
     {
       "type": "windows-restart"
     },
+    {
+      "arch": "{{user `goss_arch`}}",
+      "format": "{{user `goss_format`}}",
+      "format_options": "{{user `goss_format_options`}}",
+      "goss_file": "{{user `goss_entry_file`}}",
+      "inspect": "{{user `goss_inspect_mode`}}",
+      "target_os": "Windows",
+      "tests": [
+        "{{user `goss_tests_dir`}}"
+      ],
+      "type": "goss",
+      "url": "{{user `goss_url`}}",
+      "use_sudo": false,
+      "vars_env": {
+        "GOSS_USE_ALPHA": "1"
+      },
+      "vars_file": "{{user `goss_vars_file`}}",
+      "vars_inline": {
+        "ARCH": "amd64",
+        "OS": "{{user `distribution` | lower}}",
+        "PROVIDER": "ami",
+        "containerd_version": "{{user `containerd_version`}}",
+        "distribution_version": "{{user `distribution_version`}}",
+        "kubernetes_version": "{{user `kubernetes_semver`}}",
+        "pause_image": "{{user `pause_image`}}",
+        "runtime": "{{user `runtime`}}",
+        "windows_service_manager": "{{user `windows_service_manager`}}"
+      },
+      "version": "{{user `goss_version`}}"
+    },
     {
       "elevated_password": "{{.WinRMPassword}}",
       "elevated_user": "Administrator",

diff --git a/images/capi/packer/azure/packer-windows.json b/images/capi/packer/azure/packer-windows.json
@@ -117,6 +117,36 @@
     {
       "type": "windows-restart"
     },
+    {
+      "arch": "{{user `goss_arch`}}",
+      "format": "{{user `goss_format`}}",
+      "format_options": "{{user `goss_format_options`}}",
+      "goss_file": "{{user `goss_entry_file`}}",
+      "inspect": "{{user `goss_inspect_mode`}}",
+      "target_os": "Windows",
+      "tests": [
+        "{{user `goss_tests_dir`}}"
+      ],
+      "type": "goss",
+      "url": "{{user `goss_url`}}",
+      "use_sudo": false,
+      "vars_env": {
+        "GOSS_USE_ALPHA": "1"
+      },
+      "vars_file": "{{user `goss_vars_file`}}",
+      "vars_inline": {
+        "ARCH": "amd64",
+        "OS": "{{user `distribution` | lower}}",
+        "PROVIDER": "azure",
+        "containerd_version": "{{user `containerd_version`}}",
+        "distribution_version": "{{user `distribution_version`}}",
+        "kubernetes_version": "{{user `kubernetes_semver`}}",
+        "pause_image": "{{user `pause_image`}}",
+        "runtime": "{{user `runtime`}}",
+        "windows_service_manager": "{{user `windows_service_manager`}}"
+      },
+      "version": "{{user `goss_version`}}"
+    },
     {
       "elevated_password": "{{.WinRMPassword}}",
       "elevated_user": "packer",

diff --git a/images/capi/packer/config/goss-args.json b/images/capi/packer/config/goss-args.json
@@ -7,5 +7,5 @@
   "goss_tests_dir": "packer/goss",
   "goss_url": "",
   "goss_vars_file": "packer/goss/goss-vars.yaml",
-  "goss_version": "0.3.13"
+  "goss_version": "0.3.16"
 }
diff --git a/images/capi/packer/config/windows/goss-args.json b/images/capi/packer/config/windows/goss-args.json
@@ -0,0 +1,11 @@
+{
+  "goss_arch": "amd64",
+  "goss_entry_file": "goss/goss.yaml",
+  "goss_format": "json",
+  "goss_format_options": "pretty",
+  "goss_inspect_mode": "true",
+  "goss_tests_dir": "packer/goss",
+  "goss_url": "",
+  "goss_vars_file": "packer/goss/goss-vars.yaml",
+  "goss_version": "0.3.16"
+}
diff --git a/images/capi/packer/goss/goss-command.yaml b/images/capi/packer/goss/goss-command.yaml
@@ -1,4 +1,5 @@
 command:
+{{ if ne .Vars.OS "windows" }} # Linux Only
   containerd --version | awk -F' ' '{print substr($3,2); }':
     exit-status: 0
     stdout: []
@@ -58,6 +59,7 @@ command:
     stderr: []
     timeout: 0
 {{end}}
+{{end}}
 {{range $name, $vers := index .Vars .Vars.OS "common-command"}}
   {{ $name }}:
   {{range $key, $val := $vers}}
@@ -69,4 +71,62 @@ command:
   {{range $key, $val := $vers}}
     {{$key}}: {{$val}}
   {{end}}
+{{end}} 
+
+{{ if eq .Vars.OS "windows" }} # Windows Online
+  kubectl version:
+    exit-status: 1
+    stdout:
+    - {{.Vars.kubernetes_version}}
+    - "windows"
+    - {{.Vars.ARCH}}
+    timeout: 0
+  kubeadm version:
+    exit-status: 0
+    stdout:
+    - {{.Vars.kubernetes_version}}
+    - "windows"
+    - {{.Vars.ARCH}}
+    timeout: 0
+  kubelet --version:
+    exit-status: 0
+    stdout:
+    - {{.Vars.kubernetes_version}}
+    timeout: 0
+  Check Symbolic link to /etc/kubernetes/pki:
+    exit-status: 0
+    exec: powershell -noprofile -noninteractive -command "(Get-item -path $env:SystemDrive\var\lib\kubelet\etc\kubernetes\pki| select LinkType,Target)"
+    stdout:
+    - SymbolicLink 
+    - C:\etc\kubernetes\pki\
+{{ if eq .Vars.distribution_version "2019" }}
+  Windows build version is high enough:
+    exit-status: 0
+    exec: powershell -noprofile -noninteractive -command "(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion' -Name UBR).UBR -ge 1817"
+    stdout:
+    - "True"
+{{end}}
+{{ if eq .Vars.runtime "containerd" }}
+  containerd --version:
+    exit-status: 0
+    stdout:
+    - "{{.Vars.containerd_version}}"
+  containerd config dump:
+    exit-status: 0
+    stdout:
+    - "sandbox_image = \"{{.Vars.pause_image}}\""
+    - "conf_dir = \"C:/etc/cni/net.d\""
+    - "bin_dir = \"C:/opt/cni/bin\""
+  Check Windows Defender Exclusions are in place:
+    exit-status: 0
+    exec: powershell -noprofile -noninteractive -command "(Get-MpPreference | select ExclusionProcess)"
+    stdout:
+    - \Program Files\containerd\containerd.exe, 
+    - \Program Files\containerd\ctr.exe
+  Check SMB CompartmentNamespace Flag:
+    exit-status: 0
+    exec: powershell -noprofile -noninteractive -command "(Get-ItemProperty 'HKLM:\SYSTEM\CurrentControlSet\Services\hns\State' -Name EnableCompartmentNamespace).EnableCompartmentNamespace -eq 1"
+    stdout:
+    - True
+{{end}}
 {{end}}
diff --git a/images/capi/packer/goss/goss-files.yaml b/images/capi/packer/goss/goss-files.yaml
@@ -0,0 +1,17 @@
+file:
+{{range $name, $vers := index .Vars .Vars.OS "common-files"}}
+  {{ $name }}:
+    exists: {{ $vers.exists }}
+    filetype: {{ $vers.filetype }}
+    contains: {{ range $vers.contains}}
+    - {{.}}
+  {{end}}
+{{end}}
+{{range $name, $vers := index .Vars .Vars.OS .Vars.PROVIDER "files"}}
+  {{ $name }}:
+    exists: {{ $vers.exists }}
+    filetype: {{ $vers.filetype }}
+    contains: {{ range $vers.contains}}
+    - {{.}}
+  {{end}}
+{{end}}
diff --git a/images/capi/packer/goss/goss-service.yaml b/images/capi/packer/goss/goss-service.yaml
@@ -1,4 +1,5 @@
 service:
+{{ if ne .Vars.OS "windows"}} # Linux
   containerd:
     enabled: true
     running: true
@@ -28,4 +29,26 @@ service:
   {{range $key, $val := $vers}}
     {{$key}}: {{$val}}
   {{end}}
+{{end}}
+{{end}}
+
+{{ if eq .Vars.OS "windows"}} # Windows
+# Workaround until windows services are added to goss
+command:
+{{range $name, $vers := index .Vars .Vars.OS "common-windows-service"}}
+  "Windows Service - {{ $name }}":
+    exec: powershell -command "(Get-Service {{ $name }} | select *)"
+    exit-status: 0
+    stdout: {{range $vers.expected}}
+    - {{.}}
+    {{end}}
+{{end}}
+{{range $name, $vers := index .Vars .Vars.OS .Vars.PROVIDER "windows-service"}}
+  "Windows Service - {{ $name }}":
+    exec: powershell -command "(Get-Service {{ $name }} | select *)"
+    exit-status: 0
+    stdout: {{range $vers.expected}}
+    - {{.}}
+    {{end}}
+{{end}}
 {{end}}
diff --git a/images/capi/packer/goss/goss-vars.yaml b/images/capi/packer/goss/goss-vars.yaml
@@ -70,6 +70,12 @@ kubernetes_cni_rpm_version: ""
 # When k8s and k8s cni source is http
 kubernetes_load_additional_imgs: false
 
+#windows variables
+kubernetes_install_path: ""
+windows_service_manager: ""
+distribution_version: ""
+runtime: ""
+
 # OS Specific package/Command/Kernal Params etc...
 # Structured in below format
 # OS_NAME
@@ -175,3 +181,100 @@ photon:
       cloud-init:
       cloud-utils:
       python3-netifaces:
+
+# Windows specific variables
+windows:
+  common-files:
+    c:/etc/kubernetes/pki:
+      exists: true
+      filetype: directory
+      contains:
+    c:/var/lib/kubelet/etc/kubernetes:
+      exists: true
+      filetype: directory
+      contains:
+    c:/var/lib/kubelet/etc/kubernetes/manifests:
+      exists: true
+      filetype: directory
+      contains:
+    c:/var/log/kubelet:
+      exists: true
+      filetype: directory
+      contains:
+
+  common-windows-service:
+    cloudbase-init:
+      expected:
+      - Manual
+      - Stopped
+    kubelet:
+      expected: 
+      - Manual 
+      - Stopped
+      - "/RequiredServices.+:.+containerd/"
+    sshd:
+      expected:
+      - Automatic
+      - Running
+
+  azure:
+    windows-service:
+
+    files: 
+      'c:/program files/Cloudbase Solutions/Cloudbase-init/conf/cloudbase-init.conf': 
+        exists: true
+        filetype: file
+        contains:
+        - "logging_serial_port=COM1,115200,N,8"
+        - "metadata_services=cloudbaseinit.metadata.services.azureservice.AzureService"
+        - "cloudbaseinit.plugins.common.ephemeraldisk.EphemeralDiskPlugin"
+        - "cloudbaseinit.plugins.windows.azureguestagent.AzureGuestAgentPlugin"
+        - "cloudbaseinit.plugins.common.mtu.MTUPlugin"
+        - "cloudbaseinit.plugins.common.sethostname.SetHostNamePlugin"
+  ova:
+    windows-service:
+      vmtools:
+        expected: 
+        - Automatic 
+        - Running
+    files:
+      'c:/program files/Cloudbase Solutions/Cloudbase-init/conf/cloudbase-init.conf': 
+        exists: true
+        filetype: file
+        contains:
+        - "!/logging_serial_port=COM1,115200,N,8/"
+        - "cloudbaseinit.metadata.services.vmwareguestinfoservice.VMwareGuestInfoService"
+        - "cloudbaseinit.plugins.common.ephemeraldisk.EphemeralDiskPlugin"
+        - "cloudbaseinit.plugins.common.mtu.MTUPlugin"
+        - "cloudbaseinit.plugins.common.sethostname.SetHostNamePlugin"
+        - "cloudbaseinit.plugins.common.sshpublickeys.SetUserSSHPublicKeysPlugin"
+        - "cloudbaseinit.plugins.common.userdata.UserDataPlugin"
+        - "cloudbaseinit.plugins.common.localscripts.LocalScriptsPlugin"
+        - "cloudbaseinit.plugins.windows.createuser.CreateUserPlugin"
+        - "cloudbaseinit.plugins.windows.extendvolumes.ExtendVolumesPlugin"
+      'c:/program files/Cloudbase Solutions/Cloudbase-init/conf/cloudbase-init-unattend.conf': 
+        exists: true
+        filetype: file
+        contains:
+        - "metadata_services=cloudbaseinit.metadata.services.vmwareguestinfoservice.VMwareGuestInfoService"
+  ami:
+    windows-service:
+
+    files:
+      'c:/program files/Cloudbase Solutions/Cloudbase-init/conf/cloudbase-init.conf': 
+        exists: true
+        filetype: file
+        contains:
+        - "!/logging_serial_port=COM1,115200,N,8/"
+        - "metadata_services=cloudbaseinit.metadata.services.ec2service.EC2Service"
+        - "cloudbaseinit.plugins.common.ephemeraldisk.EphemeralDiskPlugin"
+        - "cloudbaseinit.plugins.common.mtu.MTUPlugin"
+        - "cloudbaseinit.plugins.common.sethostname.SetHostNamePlugin"
+
+
+
+
+
+
+
+