Merge pull request #59067 from chuckha/audit

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Feature Gate - Kubeadm Audit Logging

Fixes kubernetes/kubeadm#623

Signed-off-by: Chuck Ha <ha.chuck@gmail.com>



**What this PR does / why we need it**:
This PR enables [Auditing](https://kubernetes.io/docs/tasks/debug-application-cluster/audit/) behind a featureGate. A user can supply their own audit policy with configuration option as well as a place for the audit logs to live. If no policy is supplied a default policy will be provided. The default policy will log all Metadata level policy logs. It is the example provided in the documentation.
**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes kubernetes/kubeadm#623

**Special notes for your reviewer**:

**Release note**:

```release-note
kubeadm: Enable auditing behind a feature gate.
```

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -110,6 +110,11 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 					ConfigSyncPeriod: metav1.Duration{Duration: 1},
 				},
 			}
+			obj.AuditPolicyConfiguration = kubeadm.AuditPolicyConfiguration{
+				Path:      "foo",
+				LogDir:    "/foo",
+				LogMaxAge: utilpointer.Int32Ptr(0),
+			}
 		},
 		func(obj *kubeadm.NodeConfiguration, c fuzz.Continue) {
 			c.FuzzNoCustom(obj)

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -108,6 +108,9 @@ type MasterConfiguration struct {
 	// used for all control plane components.
 	UnifiedControlPlaneImage string
 
+	// AuditPolicyConfiguration defines the options for the api server audit system.
+	AuditPolicyConfiguration AuditPolicyConfiguration
+
 	// FeatureGates enabled by the user.
 	FeatureGates map[string]bool
 }
@@ -263,3 +266,14 @@ type HostPathMount struct {
 type KubeProxy struct {
 	Config *kubeproxyconfigv1alpha1.KubeProxyConfiguration
 }
+
+// AuditPolicyConfiguration holds the options for configuring the api server audit policy.
+type AuditPolicyConfiguration struct {
+	// Path is the local path to an audit policy.
+	Path string
+	// LogDir is the local path to the directory where logs should be stored.
+	LogDir string
+	// LogMaxAge is the number of days logs will be stored for. 0 indicates forever.
+	LogMaxAge *int32
+	//TODO(chuckha) add other options for audit policy.
+}

cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go

@@ -68,6 +68,12 @@ const (
 	KubeproxyKubeConfigFileName = "/var/lib/kube-proxy/kubeconfig.conf"
 )
 
+var (
+	// DefaultAuditPolicyLogMaxAge is defined as a var so its address can be taken
+	// It is the number of days to store audit logs
+	DefaultAuditPolicyLogMaxAge = int32(2)
+)
+
 func addDefaultingFuncs(scheme *runtime.Scheme) error {
 	return RegisterDefaults(scheme)
 }
@@ -117,6 +123,7 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
 		SetDefaults_KubeletConfiguration(obj)
 	}
 	SetDefaults_ProxyConfiguration(obj)
+	SetDefaults_AuditPolicyConfiguration(obj)
 }
 
 // SetDefaults_ProxyConfiguration assigns default values for the Proxy
@@ -207,3 +214,13 @@ func SetDefaults_KubeletConfiguration(obj *MasterConfiguration) {
 		scheme.Default(obj.KubeletConfiguration.BaseConfig)
 	}
 }
+
+// SetDefaults_AuditPolicyConfiguration sets default values for the AuditPolicyConfiguration
+func SetDefaults_AuditPolicyConfiguration(obj *MasterConfiguration) {
+	if obj.AuditPolicyConfiguration.LogDir == "" {
+		obj.AuditPolicyConfiguration.LogDir = constants.StaticPodAuditPolicyLogDir
+	}
+	if obj.AuditPolicyConfiguration.LogMaxAge == nil {
+		obj.AuditPolicyConfiguration.LogMaxAge = &DefaultAuditPolicyLogMaxAge
+	}
+}

cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go

@@ -100,6 +100,9 @@ type MasterConfiguration struct {
 	// be used for all control plane components.
 	UnifiedControlPlaneImage string `json:"unifiedControlPlaneImage"`
 
+	// AuditPolicyConfiguration defines the options for the api server audit system
+	AuditPolicyConfiguration AuditPolicyConfiguration `json:"auditPolicy"`
+
 	// FeatureGates enabled by the user.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
@@ -243,3 +246,14 @@ type HostPathMount struct {
 type KubeProxy struct {
 	Config *kubeproxyconfigv1alpha1.KubeProxyConfiguration `json:"config,omitempty"`
 }
+
+// AuditPolicyConfiguration holds the options for configuring the api server audit policy.
+type AuditPolicyConfiguration struct {
+	// Path is the local path to an audit policy.
+	Path string `json:"path"`
+	// LogDir is the local path to the directory where logs should be stored.
+	LogDir string `json:"logDir"`
+	// LogMaxAge is the number of days logs will be stored for. 0 indicates forever.
+	LogMaxAge *int32 `json:"logMaxAge,omitempty"`
+	//TODO(chuckha) add other options for audit policy.
+}

cmd/kubeadm/app/cmd/BUILD

@@ -47,6 +47,7 @@ go_library(
         "//cmd/kubeadm/app/preflight:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
+        "//cmd/kubeadm/app/util/audit:go_default_library",
         "//cmd/kubeadm/app/util/config:go_default_library",
         "//cmd/kubeadm/app/util/dryrun:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",

cmd/kubeadm/app/cmd/init.go

@@ -57,6 +57,7 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
+	auditutil "k8s.io/kubernetes/cmd/kubeadm/app/util/audit"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
@@ -322,6 +323,21 @@ func (i *Init) Run(out io.Writer) error {
 		fmt.Println("[externalca] The file 'ca.key' was not found, yet all other certificates are present. Using external CA mode - certificates or kubeconfig will not be generated.")
 	}
 
+	if features.Enabled(i.cfg.FeatureGates, features.Auditing) {
+		// Setup the AuditPolicy (either it was passed in and exists or it wasn't passed in and generate a default policy)
+		if i.cfg.AuditPolicyConfiguration.Path != "" {
+			// TODO(chuckha) ensure passed in audit policy is valid so users don't have to find the error in the api server log.
+			if _, err := os.Stat(i.cfg.AuditPolicyConfiguration.Path); err != nil {
+				return fmt.Errorf("error getting file info for audit policy file %q [%v]", i.cfg.AuditPolicyConfiguration.Path, err)
+			}
+		} else {
+			i.cfg.AuditPolicyConfiguration.Path = filepath.Join(kubeConfigDir, kubeadmconstants.AuditPolicyDir, kubeadmconstants.AuditPolicyFile)
+			if err := auditutil.CreateDefaultAuditLogPolicy(i.cfg.AuditPolicyConfiguration.Path); err != nil {
+				return fmt.Errorf("error creating default audit policy %q [%v]", i.cfg.AuditPolicyConfiguration.Path, err)
+			}
+		}
+	}
+
 	// Temporarily set cfg.CertificatesDir to the "real value" when writing controlplane manifests
 	// This is needed for writing the right kind of manifests
 	i.cfg.CertificatesDir = realCertsDir

cmd/kubeadm/app/cmd/upgrade/common_test.go

@@ -41,6 +41,9 @@ func TestPrintConfiguration(t *testing.T) {
 	api:
 	  advertiseAddress: ""
 	  bindPort: 0
+	auditPolicy:
+	  logDir: ""
+	  path: ""
 	certificatesDir: ""
 	cloudProvider: ""
 	etcd:
@@ -75,6 +78,9 @@ func TestPrintConfiguration(t *testing.T) {
 	api:
 	  advertiseAddress: ""
 	  bindPort: 0
+	auditPolicy:
+	  logDir: ""
+	  path: ""
 	certificatesDir: ""
 	cloudProvider: ""
 	etcd:
@@ -114,6 +120,9 @@ func TestPrintConfiguration(t *testing.T) {
 	api:
 	  advertiseAddress: ""
 	  bindPort: 0
+	auditPolicy:
+	  logDir: ""
+	  path: ""
 	certificatesDir: ""
 	cloudProvider: ""
 	etcd:

cmd/kubeadm/app/constants/constants.go

@@ -203,6 +203,19 @@ const (
 
 	// CRICtlPackage defines the go package that installs crictl
 	CRICtlPackage = "github.com/kubernetes-incubator/cri-tools/cmd/crictl"
+
+	// KubeAuditPolicyVolumeName is the name of the volume that will contain the audit policy
+	KubeAuditPolicyVolumeName = "audit"
+	// AuditPolicyDir is the directory that will contain the audit policy
+	AuditPolicyDir = "audit"
+	// AuditPolicyFile is the name of the audit policy file itself
+	AuditPolicyFile = "audit.yaml"
+	// AuditPolicyLogFile is the name of the file audit logs get written to
+	AuditPolicyLogFile = "audit.log"
+	// KubeAuditPolicyLogVolumeName is the name of the volume that will contain the audit logs
+	KubeAuditPolicyLogVolumeName = "audit-log"
+	// StaticPodAuditPolicyLogDir is the name of the directory in the static pod that will have the audit logs
+	StaticPodAuditPolicyLogDir = "/var/log/kubernetes/audit"
 )
 
 var (
@@ -311,3 +324,8 @@ func GetDNSIP(svcSubnet string) (net.IP, error) {
 
 	return dnsIP, nil
 }
+
+// GetStaticPodAuditPolicyFile returns the path to the audit policy file within a static pod
+func GetStaticPodAuditPolicyFile() string {
+	return filepath.Join(KubernetesDir, AuditPolicyDir, AuditPolicyFile)
+}

cmd/kubeadm/app/features/features.go

@@ -41,6 +41,9 @@ const (
 
 	// DynamicKubeletConfig is alpha in v1.9
 	DynamicKubeletConfig = "DynamicKubeletConfig"
+
+	// Auditing is beta in 1.8
+	Auditing = "Auditing"
 )
 
 var v190 = version.MustParseSemantic("v1.9.0-alpha.1")
@@ -53,6 +56,7 @@ var InitFeatureGates = FeatureList{
 	HighAvailability:     {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190, HiddenInHelpText: true},
 	CoreDNS:              {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190},
 	DynamicKubeletConfig: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190},
+	Auditing:             {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}},
 }
 
 // Feature represents a feature being gated

cmd/kubeadm/app/phases/controlplane/BUILD

@@ -21,6 +21,7 @@ go_test(
         "//cmd/kubeadm/app/phases/certs:go_default_library",
         "//cmd/kubeadm/test:go_default_library",
         "//pkg/master/reconcilers:go_default_library",
+        "//pkg/util/pointer:go_default_library",
         "//pkg/util/version:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
     ],

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -224,6 +224,16 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio
 		command = append(command, "--feature-gates=DynamicKubeletConfig=true")
 	}
 
+	if features.Enabled(cfg.FeatureGates, features.Auditing) {
+		command = append(command, "--audit-policy-file="+kubeadmconstants.GetStaticPodAuditPolicyFile())
+		command = append(command, "--audit-log-path="+filepath.Join(kubeadmconstants.StaticPodAuditPolicyLogDir, kubeadmconstants.AuditPolicyLogFile))
+		if cfg.AuditPolicyConfiguration.LogMaxAge == nil {
+			command = append(command, fmt.Sprintf("--audit-log-maxage=%d", kubeadmapiext.DefaultAuditPolicyLogMaxAge))
+		} else {
+			command = append(command, fmt.Sprintf("--audit-log-maxage=%d", *cfg.AuditPolicyConfiguration.LogMaxAge))
+		}
+	}
+
 	return command
 }