Skip to content

Commit

Permalink
HOSTEDCP-839: Audit log sidecars for openshift-apiserver and openshif…
Browse files Browse the repository at this point in the history 
…t-oauth-apiserver

This adds a tail sidecar container for the audit logs for the above
deployments. It also plumbs the audit config into the two deployments
and sets up the webhook configuration for audit logs.

Fixes: HOSTEDCP-839
  • Loading branch information
@imain
imain authored and openshift-cherrypick-robot committed Mar 21, 2023
1 parent f4a73f2 commit 508cfc2
Show file tree
Hide file tree
Showing 7 changed files with 186 additions and 76 deletions.
12 changes: 6 additions & 6 deletions control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go
View file
Expand Up @@ -2284,14 +2284,14 @@ func (r *HostedControlPlaneReconciler) reconcileOpenShiftAPIServer(ctx context.C
p := oapi.NewOpenShiftAPIServerParams(hcp, observedConfig, releaseImage.ComponentImages(), r.SetDefaultSecurityContext)
oapicfg := manifests.OpenShiftAPIServerConfig(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, oapicfg, func() error {
return oapi.ReconcileConfig(oapicfg, p.OwnerRef, p.EtcdURL, p.IngressDomain(), p.MinTLSVersion(), p.CipherSuites(), p.Image, p.Project)
return oapi.ReconcileConfig(oapicfg, p.AuditWebhookRef, p.OwnerRef, p.EtcdURL, p.IngressDomain(), p.MinTLSVersion(), p.CipherSuites(), p.Image, p.Project)
}); err != nil {
return fmt.Errorf("failed to reconcile openshift apiserver config: %w", err)
}

auditCfg := manifests.OpenShiftAPIServerAuditConfig(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, auditCfg, func() error {
return oapi.ReconcileAuditConfig(auditCfg, p.OwnerRef)
return oapi.ReconcileAuditConfig(auditCfg, p.OwnerRef, p.AuditPolicyConfig())
}); err != nil {
return fmt.Errorf("failed to reconcile openshift apiserver audit config: %w", err)
}
Expand All @@ -2316,7 +2316,7 @@ func (r *HostedControlPlaneReconciler) reconcileOpenShiftAPIServer(ctx context.C

deployment := manifests.OpenShiftAPIServerDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, deployment, func() error {
return oapi.ReconcileDeployment(deployment, p.OwnerRef, oapicfg, p.OpenShiftAPIServerDeploymentConfig, p.OpenShiftAPIServerImage, p.ProxyImage, p.EtcdURL, p.AvailabilityProberImage, util.APIPort(hcp))
return oapi.ReconcileDeployment(deployment, p.AuditWebhookRef, p.OwnerRef, oapicfg, p.OpenShiftAPIServerDeploymentConfig, p.OpenShiftAPIServerImage, p.ProxyImage, p.EtcdURL, p.AvailabilityProberImage, util.APIPort(hcp))
}); err != nil {
return fmt.Errorf("failed to reconcile openshift apiserver deployment: %w", err)
}
Expand All @@ -2328,14 +2328,14 @@ func (r *HostedControlPlaneReconciler) reconcileOpenShiftOAuthAPIServer(ctx cont
p := oapi.NewOpenShiftAPIServerParams(hcp, observedConfig, releaseImage.ComponentImages(), r.SetDefaultSecurityContext)
auditCfg := manifests.OpenShiftOAuthAPIServerAuditConfig(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, auditCfg, func() error {
return oapi.ReconcileAuditConfig(auditCfg, p.OwnerRef)
return oapi.ReconcileAuditConfig(auditCfg, p.OwnerRef, p.AuditPolicyConfig())
}); err != nil {
return fmt.Errorf("failed to reconcile openshift oauth apiserver audit config: %w", err)
}

pdb := manifests.OpenShiftOAuthAPIServerDisruptionBudget(hcp.Namespace)
if result, err := createOrUpdate(ctx, r, pdb, func() error {
return oapi.ReconcileOpenShiftOAuthAPIServerPodDisruptionBudget(pdb, p.OAuthAPIServerDeploymentParams())
return oapi.ReconcileOpenShiftOAuthAPIServerPodDisruptionBudget(pdb, p.OAuthAPIServerDeploymentParams(hcp))
}); err != nil {
return fmt.Errorf("failed to reconcile openshift oauth apiserver pdb: %w", err)
} else {
Expand All @@ -2344,7 +2344,7 @@ func (r *HostedControlPlaneReconciler) reconcileOpenShiftOAuthAPIServer(ctx cont

deployment := manifests.OpenShiftOAuthAPIServerDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, deployment, func() error {
return oapi.ReconcileOAuthAPIServerDeployment(deployment, p.OwnerRef, p.OAuthAPIServerDeploymentParams(), util.APIPort(hcp))
return oapi.ReconcileOAuthAPIServerDeployment(deployment, p.OwnerRef, p.OAuthAPIServerDeploymentParams(hcp), util.APIPort(hcp))
}); err != nil {
return fmt.Errorf("failed to reconcile openshift oauth apiserver deployment: %w", err)
}
Expand Down
4 changes: 2 additions & 2 deletions control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go
View file
Expand Up @@ -155,7 +155,7 @@ func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment,
"/usr/bin/tail",
"-c+1",
"-F",
"/var/log/kube-apiserver/audit.log",
fmt.Sprintf("%s/%s", volumeMounts.Path(kasContainerMain().Name, kasVolumeWorkLogs().Name), "audit.log"),
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
Expand All @@ -165,7 +165,7 @@ func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment,
},
VolumeMounts: []corev1.VolumeMount{{
Name: kasVolumeWorkLogs().Name,
MountPath: "/var/log/kube-apiserver",
MountPath: volumeMounts.Path(kasContainerMain().Name, kasVolumeWorkLogs().Name),
}},
},
},
Expand Down
70 changes: 9 additions & 61 deletions control-plane-operator/controllers/hostedcontrolplane/oapi/auditcfg.go
View file
@@ -1,84 +1,32 @@
package oapi

import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
"fmt"

oauthv1 "github.com/openshift/api/oauth/v1"
configv1 "github.com/openshift/api/config/v1"
corev1 "k8s.io/api/core/v1"

"github.com/openshift/hypershift/support/config"
"github.com/openshift/library-go/pkg/operator/apiserver/audit"
)

const (
auditPolicyConfigMapKey = "policy.yaml"
)

func ReconcileAuditConfig(cm *corev1.ConfigMap, ownerRef config.OwnerRef) error {
func ReconcileAuditConfig(cm *corev1.ConfigMap, ownerRef config.OwnerRef, auditConfig configv1.Audit) error {
ownerRef.ApplyTo(cm)
if cm.Data == nil {
cm.Data = map[string]string{}
}
policy := defaultAuditPolicy()
policy, err := audit.GetAuditPolicy(auditConfig)
if err != nil {
return fmt.Errorf("failed to get audit policy: %w", err)
}
policyBytes, err := config.SerializeAuditPolicy(policy)
if err != nil {
return err
}
cm.Data[auditPolicyConfigMapKey] = string(policyBytes)
return nil
}

func defaultAuditPolicy() *auditv1.Policy {
return &auditv1.Policy{
TypeMeta: metav1.TypeMeta{
Kind: "Policy",
APIVersion: auditv1.SchemeGroupVersion.String(),
},
OmitStages: []auditv1.Stage{
auditv1.StageRequestReceived,
},
Rules: []auditv1.PolicyRule{
{
Level: auditv1.LevelNone,
Resources: []auditv1.GroupResources{
{
Group: corev1.SchemeGroupVersion.Group,
Resources: []string{
"events",
},
},
},
},
{
Level: auditv1.LevelNone,
Resources: []auditv1.GroupResources{
{
Group: oauthv1.SchemeGroupVersion.Group,
Resources: []string{
"oauthaccesstokens",
"oauthauthorizetokens",
},
},
},
},
{
Level: auditv1.LevelNone,
NonResourceURLs: []string{
"/api*",
"/version",
"/healthz",
},
UserGroups: []string{
"system:authenticated",
"system:unauthenticated",
},
},
{
Level: auditv1.LevelMetadata,
OmitStages: []auditv1.Stage{
auditv1.StageRequestReceived,
},
},
},
}
}
18 changes: 15 additions & 3 deletions control-plane-operator/controllers/hostedcontrolplane/oapi/config.go
View file
Expand Up @@ -9,6 +9,7 @@ import (

configv1 "github.com/openshift/api/config/v1"
openshiftcpv1 "github.com/openshift/api/openshiftcontrolplane/v1"
hyperv1 "github.com/openshift/hypershift/api/v1beta1"

"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common"
"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas"
Expand All @@ -25,7 +26,7 @@ const (
defaultInternalRegistryHostname = "image-registry.openshift-image-registry.svc:5000"
)

func ReconcileConfig(cm *corev1.ConfigMap, ownerRef config.OwnerRef, etcdURL, ingressDomain, minTLSVersion string, cipherSuites []string, imageConfig *configv1.Image, projectConfig *configv1.Project) error {
func ReconcileConfig(cm *corev1.ConfigMap, auditWebhookRef *corev1.LocalObjectReference, ownerRef config.OwnerRef, etcdURL, ingressDomain, minTLSVersion string, cipherSuites []string, imageConfig *configv1.Image, projectConfig *configv1.Project) error {
ownerRef.ApplyTo(cm)
if cm.Data == nil {
cm.Data = map[string]string{}
Expand All @@ -36,7 +37,7 @@ func ReconcileConfig(cm *corev1.ConfigMap, ownerRef config.OwnerRef, etcdURL, in
return fmt.Errorf("failed to read existing config: %w", err)
}
}
reconcileConfigObject(openshiftAPIServerConfig, etcdURL, ingressDomain, minTLSVersion, cipherSuites, imageConfig, projectConfig)
reconcileConfigObject(openshiftAPIServerConfig, auditWebhookRef, etcdURL, ingressDomain, minTLSVersion, cipherSuites, imageConfig, projectConfig)
serializedConfig, err := util.SerializeResource(openshiftAPIServerConfig, api.Scheme)
if err != nil {
return fmt.Errorf("failed to serialize openshift apiserver config: %w", err)
Expand All @@ -45,7 +46,7 @@ func ReconcileConfig(cm *corev1.ConfigMap, ownerRef config.OwnerRef, etcdURL, in
return nil
}

func reconcileConfigObject(cfg *openshiftcpv1.OpenShiftAPIServerConfig, etcdURL, ingressDomain, minTLSVersion string, cipherSuites []string, imageConfig *configv1.Image, projectConfig *configv1.Project) {
func reconcileConfigObject(cfg *openshiftcpv1.OpenShiftAPIServerConfig, auditWebhookRef *corev1.LocalObjectReference, etcdURL, ingressDomain, minTLSVersion string, cipherSuites []string, imageConfig *configv1.Image, projectConfig *configv1.Project) {
cfg.TypeMeta = metav1.TypeMeta{
Kind: "OpenShiftAPIServerConfig",
APIVersion: openshiftcpv1.GroupVersion.String(),
Expand All @@ -61,6 +62,12 @@ func reconcileConfigObject(cfg *openshiftcpv1.OpenShiftAPIServerConfig, etcdURL,
"audit-policy-file": {cpath(oasVolumeAuditConfig().Name, auditPolicyConfigMapKey)},
"audit-log-path": {cpath(oasVolumeWorkLogs().Name, "audit.log")},
}

if auditWebhookRef != nil {
cfg.APIServerArguments["audit-webhook-config-file"] = []string{auditWebhookConfigFile()}
cfg.APIServerArguments["audit-webhook-mode"] = []string{"batch"}
}

cfg.KubeClientConfig.KubeConfig = cpath(oasVolumeKubeconfig().Name, kas.KubeconfigKey)
cfg.ServingInfo = configv1.HTTPServingInfo{
ServingInfo: configv1.ServingInfo{
Expand Down Expand Up @@ -111,3 +118,8 @@ func reconcileConfigObject(cfg *openshiftcpv1.OpenShiftAPIServerConfig, etcdURL,
},
}
}

func auditWebhookConfigFile() string {
cfgDir := oasAuditWebhookConfigFileVolumeMount.Path(oasContainerMain().Name, oasAuditWebhookConfigFileVolume().Name)
return path.Join(cfgDir, hyperv1.AuditWebhookKubeconfigKey)
}
62 changes: 61 additions & 1 deletion control-plane-operator/controllers/hostedcontrolplane/oapi/deployment.go
View file
Expand Up @@ -55,6 +55,12 @@ var (
oasVolumeKonnectivityProxyCA().Name: "/etc/konnectivity/proxy-ca",
},
}

oasAuditWebhookConfigFileVolumeMount = util.PodVolumeMounts{
oasContainerMain().Name: {
oasAuditWebhookConfigFileVolume().Name: "/etc/kubernetes/auditwebhook",
},
}
)

func openShiftAPIServerLabels() map[string]string {
Expand All @@ -64,7 +70,7 @@ func openShiftAPIServerLabels() map[string]string {
}
}

func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef, config *corev1.ConfigMap, deploymentConfig config.DeploymentConfig, image string, socks5ProxyImage string, etcdURL string, availabilityProberImage string, apiPort *int32) error {
func ReconcileDeployment(deployment *appsv1.Deployment, auditWebhookRef *corev1.LocalObjectReference, ownerRef config.OwnerRef, config *corev1.ConfigMap, deploymentConfig config.DeploymentConfig, image string, socks5ProxyImage string, etcdURL string, availabilityProberImage string, apiPort *int32) error {
ownerRef.ApplyTo(deployment)

// preserve existing resource requirements for main OAS container
Expand Down Expand Up @@ -109,6 +115,27 @@ func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef
InitContainers: []corev1.Container{util.BuildContainer(oasTrustAnchorGenerator(), buildOASTrustAnchorGenerator(image))},
Containers: []corev1.Container{
util.BuildContainer(oasContainerMain(), buildOASContainerMain(image, strings.Split(etcdUrlData.Host, ":")[0], defaultOAPIPort)),
{
Name: "audit-logs",
Image: image,
ImagePullPolicy: corev1.PullIfNotPresent,
Command: []string{
"/usr/bin/tail",
"-c+1",
"-F",
fmt.Sprintf("%s/%s", volumeMounts.Path(oasContainerMain().Name, oasVolumeWorkLogs().Name), "audit.log"),
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("5m"),
corev1.ResourceMemory: resource.MustParse("10Mi"),
},
},
VolumeMounts: []corev1.VolumeMount{{
Name: oasVolumeWorkLogs().Name,
MountPath: volumeMounts.Path(oasContainerMain().Name, oasVolumeWorkLogs().Name),
}},
},
util.BuildContainer(oasSocks5ProxyContainer(), buildOASSocks5ProxyContainer(socks5ProxyImage)),
},
Volumes: []corev1.Volume{
Expand Down Expand Up @@ -137,6 +164,10 @@ func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef
},
}

if auditWebhookRef != nil {
applyOASAuditWebhookConfigFileVolume(&deployment.Spec.Template.Spec, auditWebhookRef)
}

util.AvailabilityProber(kas.InClusterKASReadyURL(deployment.Namespace, apiPort), availabilityProberImage, &deployment.Spec.Template.Spec)

deploymentConfig.ApplyTo(deployment)
Expand Down Expand Up @@ -272,6 +303,35 @@ func buildOASVolumeAuditConfig(v *corev1.Volume) {
v.ConfigMap.Name = manifests.OpenShiftAPIServerAuditConfig("").Name
}

func oasAuditWebhookConfigFileVolume() *corev1.Volume {
return &corev1.Volume{
Name: "oas-audit-webhook",
}
}

func buildOASAuditWebhookConfigFileVolume(auditWebhookRef *corev1.LocalObjectReference) func(v *corev1.Volume) {
return func(v *corev1.Volume) {
v.Secret = &corev1.SecretVolumeSource{}
v.Secret.SecretName = auditWebhookRef.Name
}
}

func applyOASAuditWebhookConfigFileVolume(podSpec *corev1.PodSpec, auditWebhookRef *corev1.LocalObjectReference) {
podSpec.Volumes = append(podSpec.Volumes, util.BuildVolume(oasAuditWebhookConfigFileVolume(), buildOASAuditWebhookConfigFileVolume(auditWebhookRef)))
var container *corev1.Container
for i, c := range podSpec.Containers {
if c.Name == oasContainerMain().Name {
container = &podSpec.Containers[i]
break
}
}
if container == nil {
panic("main openshift apiserver container not found in spec")
}
container.VolumeMounts = append(container.VolumeMounts,
oasAuditWebhookConfigFileVolumeMount.ContainerMounts(oasContainerMain().Name)...)
}

func oasVolumeKubeconfig() *corev1.Volume {
return &corev1.Volume{
Name: "kubeconfig",
Expand Down

0 comments on commit 508cfc2

Please sign in to comment.