Skip to content

Commit

Permalink
Add a new FIPS test
Browse files Browse the repository at this point in the history
See discussion in
openshift/release#10488
and https://bugzilla.redhat.com/show_bug.cgi?id=1861095

This test replaces the bash code in the release repo
with a more proper test here.

While here I noticed that the topology tests had some code
that reused the MCD as a handy privileged pod; extract
that to the toplevel utils and use both here and there.
  • Loading branch information
cgwalters committed Sep 22, 2020
1 parent 4528f69 commit be9c0cf
Show file tree
Hide file tree
Showing 4 changed files with 136 additions and 40 deletions.
87 changes: 87 additions & 0 deletions test/extended/security/fips.go
@@ -0,0 +1,87 @@
package security

import (
"context"
"fmt"
"strconv"
"strings"

g "github.com/onsi/ginkgo"
o "github.com/onsi/gomega"
exutil "github.com/openshift/origin/test/extended/util"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
"sigs.k8s.io/yaml"
)

const (
installConfigName = "cluster-config-v1"
fipsFile = "/proc/sys/crypto/fips_enabled"
)

// installConfig The subset of openshift-install's InstallConfig we parse for this test
type installConfig struct {
FIPS bool `json:"fips,omitempty"`
}

func installConfigFromCluster(client clientcorev1.ConfigMapsGetter) (*installConfig, error) {
cm, err := client.ConfigMaps("kube-system").Get(context.Background(), installConfigName, metav1.GetOptions{})
if err != nil {
return nil, err
}
data, ok := cm.Data["install-config"]
if !ok {
return nil, fmt.Errorf("No install-config found in kube-system/%s", installConfigName)
}
config := &installConfig{}
if err := yaml.Unmarshal([]byte(data), config); err != nil {
return nil, err
}
return config, nil
}

func validateFIPSOnNode(oc *exutil.CLI, fipsExpected bool, node *corev1.Node) error {
command := []string{"cat", fipsFile}
out, err := exutil.ExecCommandOnMachineConfigDaemon(oc.AdminKubeClient(), oc, node, command)
if err != nil {
return err
}
nodeFips, err := strconv.ParseBool(strings.TrimSuffix(string(out), "\n"))
if err != nil {
return fmt.Errorf("Error parsing %s on node %s: %v", fipsFile, node.Name, err)
}
if nodeFips != fipsExpected {
return fmt.Errorf("Expected FIPS state %v, found %v", fipsExpected, nodeFips)
}
return nil
}

var _ = g.Describe("[sig-arch] [Conformance] FIPS", func() {
defer g.GinkgoRecover()
oc := exutil.NewCLI("fips")

g.It("TestFIPS", func() {
clusterAdminKubeClientset := oc.AdminKubeClient()
installConfig, err := installConfigFromCluster(clusterAdminKubeClientset.CoreV1())
o.Expect(err).NotTo(o.HaveOccurred())

// fetch one control plane and one worker, and validate FIPS state on it
masterNodes, err := clusterAdminKubeClientset.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{
LabelSelector: "node-role.kubernetes.io/master",
})
o.Expect(err).NotTo(o.HaveOccurred())
masterNode := &masterNodes.Items[0]
err = validateFIPSOnNode(oc, installConfig.FIPS, masterNode)
o.Expect(err).NotTo(o.HaveOccurred())
workerNodes, err := clusterAdminKubeClientset.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{
LabelSelector: "node-role.kubernetes.io/worker",
})
o.Expect(err).NotTo(o.HaveOccurred())
if len(workerNodes.Items) > 0 {
workerNode := &workerNodes.Items[0]
err = validateFIPSOnNode(oc, installConfig.FIPS, workerNode)
o.Expect(err).NotTo(o.HaveOccurred())
}
})
})
42 changes: 2 additions & 40 deletions test/extended/topology_manager/utils.go
Expand Up @@ -15,7 +15,6 @@ import (
exutil "github.com/openshift/origin/test/extended/util"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
Expand All @@ -40,9 +39,6 @@ const (
// no default for sriovNetworkNamespace: use the e2e test framework default
defaultSriovNetwork = "sriov-network"
defaultIPFamily = "v4"

namespaceMachineConfigOperator = "openshift-machine-config-operator"
containerMachineConfigDaemon = "machine-config-daemon"
)

const (
Expand Down Expand Up @@ -118,23 +114,6 @@ func getNodeByRole(c clientset.Interface, role string) ([]corev1.Node, error) {
return nodes.Items, nil
}

func getMachineConfigDaemonByNode(c clientset.Interface, node *corev1.Node) (*corev1.Pod, error) {
listOptions := metav1.ListOptions{
FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": node.Name}).String(),
LabelSelector: labels.SelectorFromSet(labels.Set{"k8s-app": "machine-config-daemon"}).String(),
}

mcds, err := c.CoreV1().Pods(namespaceMachineConfigOperator).List(context.Background(), listOptions)
if err != nil {
return nil, err
}

if len(mcds.Items) < 1 {
return nil, fmt.Errorf("failed to get machine-config-daemon pod for the node %q", node.Name)
}
return &mcds.Items[0], nil
}

const (
sysFSNumaNodePath = "/sys/devices/system/node"
)
Expand Down Expand Up @@ -196,27 +175,10 @@ func makeAllowedCpuListEnv(out string) string {
return fmt.Sprintf("CPULIST_ALLOWED=%s\n", strings.TrimSpace(pair[1]))
}

// execCommandOnMachineConfigDaemon returns the output of the command execution on the machine-config-daemon pod that runs on the specified node
func execCommandOnMachineConfigDaemon(c clientset.Interface, oc *exutil.CLI, node *corev1.Node, command []string) (string, error) {
mcd, err := getMachineConfigDaemonByNode(c, node)
if err != nil {
return "", err
}

initialArgs := []string{
"-n", namespaceMachineConfigOperator,
"-c", containerMachineConfigDaemon,
"--request-timeout", "30",
mcd.Name,
}
args := append(initialArgs, command...)
return oc.AsAdmin().Run("rsh").Args(args...).Output()
}

// getKubeletConfig returns KubeletConfiguration loaded from the node /etc/kubernetes/kubelet.conf
func getKubeletConfig(c clientset.Interface, oc *exutil.CLI, node *corev1.Node) (*kubeletconfigv1beta1.KubeletConfiguration, error) {
command := []string{"cat", path.Join("/rootfs", filePathKubeletConfig)}
kubeletData, err := execCommandOnMachineConfigDaemon(c, oc, node, command)
kubeletData, err := exutil.ExecCommandOnMachineConfigDaemon(c, oc, node, command)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -249,7 +211,7 @@ func parseSysfsNodeOnline(data string) (int, error) {

func getNumaNodeCountFromNode(c clientset.Interface, oc *exutil.CLI, node *corev1.Node) (int, error) {
command := []string{"cat", "/sys/devices/system/node/online"}
out, err := execCommandOnMachineConfigDaemon(c, oc, node, command)
out, err := exutil.ExecCommandOnMachineConfigDaemon(c, oc, node, command)
if err != nil {
return 0, err
}
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

45 changes: 45 additions & 0 deletions test/extended/util/pods.go
Expand Up @@ -2,18 +2,28 @@ package util

import (
"context"
"fmt"
"strings"
"time"

corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
kutilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
clientset "k8s.io/client-go/kubernetes"
e2e "k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/pod"
)

const (
namespaceMachineConfigOperator = "openshift-machine-config-operator"
containerMachineConfigDaemon = "machine-config-daemon"
)

// WaitForNoPodsAvailable waits until there are no pods in the
// given namespace
func WaitForNoPodsAvailable(oc *CLI) error {
Expand Down Expand Up @@ -65,3 +75,38 @@ func CreateCentosExecPodOrFail(client kubernetes.Interface, ns, generateName str
}
})
}

// getMachineConfigDaemonByNode finds the privileged daemonset from the Machine Config Operator
func getMachineConfigDaemonByNode(c clientset.Interface, node *corev1.Node) (*corev1.Pod, error) {
listOptions := metav1.ListOptions{
FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": node.Name}).String(),
LabelSelector: labels.SelectorFromSet(labels.Set{"k8s-app": "machine-config-daemon"}).String(),
}

mcds, err := c.CoreV1().Pods(namespaceMachineConfigOperator).List(context.Background(), listOptions)
if err != nil {
return nil, err
}

if len(mcds.Items) < 1 {
return nil, fmt.Errorf("failed to get machine-config-daemon pod for the node %q", node.Name)
}
return &mcds.Items[0], nil
}

// ExecCommandOnMachineConfigDaemon returns the output of the command execution on the machine-config-daemon pod that runs on the specified node
func ExecCommandOnMachineConfigDaemon(c clientset.Interface, oc *CLI, node *corev1.Node, command []string) (string, error) {
mcd, err := getMachineConfigDaemonByNode(c, node)
if err != nil {
return "", err
}

initialArgs := []string{
"-n", namespaceMachineConfigOperator,
"-c", containerMachineConfigDaemon,
"--request-timeout", "30",
mcd.Name,
}
args := append(initialArgs, command...)
return oc.AsAdmin().Run("rsh").Args(args...).Output()
}

0 comments on commit be9c0cf

Please sign in to comment.