From e4f2c320468c3769155773d9a401899fef12b350 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 15 Apr 2019 16:20:41 -0700 Subject: [PATCH] chore(CIS): enforce CIS modprobe recommendations --- packer/install-dependencies.sh | 3 ++ packer/vhd-image-builder.json | 5 +++ parts/k8s/cloud-init/artifacts/cis.sh | 1 + .../cloud-init/artifacts/modprobe-CIS.conf | 8 +++++ parts/k8s/cloud-init/masternodecustomdata.yml | 7 +++++ parts/k8s/cloud-init/nodecustomdata.yml | 7 +++++ pkg/engine/armvariables.go | 1 + pkg/engine/armvariables_test.go | 3 ++ pkg/engine/const.go | 1 + test/e2e/kubernetes/kubernetes_test.go | 31 +++++++++++++++++++ .../scripts/modprobe-config-validate.sh | 11 +++++++ 11 files changed, 78 insertions(+) create mode 100644 parts/k8s/cloud-init/artifacts/modprobe-CIS.conf create mode 100755 test/e2e/kubernetes/scripts/modprobe-config-validate.sh diff --git a/packer/install-dependencies.sh b/packer/install-dependencies.sh index effb31bb77..936a3e8449 100644 --- a/packer/install-dependencies.sh +++ b/packer/install-dependencies.sh @@ -10,6 +10,8 @@ ETC_ISSUE_CONFIG_SRC=/home/packer/etc-issue ETC_ISSUE_CONFIG_DEST=/etc/issue ETC_ISSUE_NET_CONFIG_SRC=/home/packer/etc-issue.net ETC_ISSUE_NET_CONFIG_DEST=/etc/issue.net +MODPROBE_CIS_SRC=/home/packer/modprobe-CIS.conf +MODPROBE_CIS_DEST=/etc/modprobe.d/CIS.conf echo "Starting build on " `date` > ${RELEASE_NOTES_FILEPATH} echo "Using kernel:" >> ${RELEASE_NOTES_FILEPATH} @@ -18,6 +20,7 @@ cp $SYSCTL_CONFIG_SRC $SYSCTL_CONFIG_DEST sysctl_reload 20 5 10 cp $ETC_ISSUE_CONFIG_SRC $ETC_ISSUE_CONFIG_DEST cp $ETC_ISSUE_NET_CONFIG_SRC $ETC_ISSUE_NET_CONFIG_DEST +cp $MODPROBE_CIS_SRC $MODPROBE_CIS_DEST echo "" echo "Components downloaded in this VHD build (some of the below components might get deleted during cluster provisioning if they are not needed):" >> ${RELEASE_NOTES_FILEPATH} diff --git a/packer/vhd-image-builder.json b/packer/vhd-image-builder.json index 97c7220441..5cfd2aab83 100644 --- a/packer/vhd-image-builder.json +++ b/packer/vhd-image-builder.json @@ -84,6 +84,11 @@ "source": "parts/k8s/cloud-init/artifacts/etc-issue.net", "destination": "/home/packer/etc-issue.net" }, + { + "type": "file", + "source": "parts/k8s/cloud-init/artifacts/modprobe-CIS.conf", + "destination": "/home/packer/modprobe-CIS.conf" + }, { "type": "shell", "inline": [ diff --git a/parts/k8s/cloud-init/artifacts/cis.sh b/parts/k8s/cloud-init/artifacts/cis.sh index 2d7efc7a0e..4b241fcd44 100644 --- a/parts/k8s/cloud-init/artifacts/cis.sh +++ b/parts/k8s/cloud-init/artifacts/cis.sh @@ -45,6 +45,7 @@ assignFilePermissions() { chmod 600 /etc/shadow- || exit $ERR_CIS_ASSIGN_FILE_PERMISSION chmod 600 /etc/group- || exit $ERR_CIS_ASSIGN_FILE_PERMISSION chmod 644 /etc/sysctl.d/60-CIS.conf || exit $ERR_CIS_ASSIGN_FILE_PERMISSION + chmod 644 /etc/modprobe.d/CIS.conf || exit $ERR_CIS_ASSIGN_FILE_PERMISSION } applyCIS() { diff --git a/parts/k8s/cloud-init/artifacts/modprobe-CIS.conf b/parts/k8s/cloud-init/artifacts/modprobe-CIS.conf new file mode 100644 index 0000000000..5f56a2662b --- /dev/null +++ b/parts/k8s/cloud-init/artifacts/modprobe-CIS.conf @@ -0,0 +1,8 @@ +# 3.5.1 Ensure DCCP is disabled +install dccp /bin/true +# 3.5.2 Ensure SCTP is disabled +install sctp /bin/true +# 3.5.3 Ensure RDS is disabled +install rds /bin/true +# 3.5.4 Ensure TIPC is disabled +install tipc /bin/true \ No newline at end of file diff --git a/parts/k8s/cloud-init/masternodecustomdata.yml b/parts/k8s/cloud-init/masternodecustomdata.yml index 7696b7b237..200be16b3d 100644 --- a/parts/k8s/cloud-init/masternodecustomdata.yml +++ b/parts/k8s/cloud-init/masternodecustomdata.yml @@ -146,6 +146,13 @@ write_files: owner: root content: !!binary | {{CloudInitData "cisNetEnforcement"}} + +- path: /etc/modprobe.d/CIS.conf + permissions: "0644" + encoding: gzip + owner: root + content: !!binary | + {{CloudInitData "modprobeConfCIS"}} {{end}} {{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}} diff --git a/parts/k8s/cloud-init/nodecustomdata.yml b/parts/k8s/cloud-init/nodecustomdata.yml index 53ffe40bfb..bc5dec41d9 100644 --- a/parts/k8s/cloud-init/nodecustomdata.yml +++ b/parts/k8s/cloud-init/nodecustomdata.yml @@ -140,6 +140,13 @@ write_files: owner: root content: !!binary | {{CloudInitData "cisNetEnforcement"}} + +- path: /etc/modprobe.d/CIS.conf + permissions: "0644" + encoding: gzip + owner: root + content: !!binary | + {{CloudInitData "modprobeConfCIS"}} {{end}} {{if .KubernetesConfig.RequiresDocker}} diff --git a/pkg/engine/armvariables.go b/pkg/engine/armvariables.go index 10abc0a265..135c92499b 100644 --- a/pkg/engine/armvariables.go +++ b/pkg/engine/armvariables.go @@ -128,6 +128,7 @@ func getK8sMasterVars(cs *api.ContainerService) (map[string]interface{}, error) "etcIssue": getBase64EncodedGzippedCustomScript(etcIssue), "etcIssueNet": getBase64EncodedGzippedCustomScript(etcIssueNet), "cisNetEnforcement": getBase64EncodedGzippedCustomScript(cisNetEnforcement), + "modprobeConfCIS": getBase64EncodedGzippedCustomScript(modprobeConfCIS), }, "provisionScriptParametersCommon": fmt.Sprintf("[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' CONTAINERD_VERSION=',parameters('containerdVersion'),' MOBY_VERSION=',parameters('mobyVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION=%s HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiServerCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' NETWORK_POLICY=',parameters('networkPolicy'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' MAXIMUM_LOADBALANCER_RULE_COUNT=',variables('maximumLoadBalancerRuleCount'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' IS_HOSTED_MASTER=%t',' PRIVATE_AZURE_REGISTRY_SERVER=',parameters('privateAzureRegistryServer'),' AUTHENTICATION_METHOD=',variables('customCloudAuthenticationMethod'),' IDENTITY_SYSTEM=',variables('customCloudIdentifySystem'))]", orchProfile.OrchestratorVersion, isHostedMaster), diff --git a/pkg/engine/armvariables_test.go b/pkg/engine/armvariables_test.go index b72aafca89..2070b7bf2b 100644 --- a/pkg/engine/armvariables_test.go +++ b/pkg/engine/armvariables_test.go @@ -149,6 +149,7 @@ func TestK8sVars(t *testing.T) { "etcIssue": getBase64EncodedGzippedCustomScript(etcIssue), "etcIssueNet": getBase64EncodedGzippedCustomScript(etcIssueNet), "cisNetEnforcement": getBase64EncodedGzippedCustomScript(cisNetEnforcement), + "modprobeConfCIS": getBase64EncodedGzippedCustomScript(modprobeConfCIS), }, "provisionScriptParametersCommon": fmt.Sprintf("[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' CONTAINERD_VERSION=',parameters('containerdVersion'),' MOBY_VERSION=',parameters('mobyVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION=%s HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiServerCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' NETWORK_POLICY=',parameters('networkPolicy'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' MAXIMUM_LOADBALANCER_RULE_COUNT=',variables('maximumLoadBalancerRuleCount'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' IS_HOSTED_MASTER=false',' PRIVATE_AZURE_REGISTRY_SERVER=',parameters('privateAzureRegistryServer'),' AUTHENTICATION_METHOD=',variables('customCloudAuthenticationMethod'),' IDENTITY_SYSTEM=',variables('customCloudIdentifySystem'))]", testK8sVersion), "provisionScriptParametersMaster": "[concat('COSMOS_URI= MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true NO_OUTBOUND=false CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", @@ -522,6 +523,7 @@ func TestK8sVars(t *testing.T) { "etcIssue": getBase64EncodedGzippedCustomScript(etcIssue), "etcIssueNet": getBase64EncodedGzippedCustomScript(etcIssueNet), "cisNetEnforcement": getBase64EncodedGzippedCustomScript(cisNetEnforcement), + "modprobeConfCIS": getBase64EncodedGzippedCustomScript(modprobeConfCIS), }, "masterPublicLbFQDN": "blueorange.local.cloudapp.azurestack.external", "provisionConfigsCustomCloud": "H4sIAAAAAAAA/7RX7XLaSBb9z1PcyGzZrlgI72ZdKe+SlCKUCRMbXJKcVCak5Lb6CjoW3Up3yx9j8+5TLRBgwIakZvhhG3E559zT96O988K5ZNy5JGpYqyFXhUQPpWYpS4hGtbcP9zUAAPeP88API9f7GPvdT52g1z31u1H8e9jrxmdu9KFlOagT56q4RMlRo3LIn4VEpUlylWSioI3vSnBrGSvww9554Pnxqdt1f/OD2O+2z3qdbtS6SIiG+iZaeIDvP6AhUYlCJnhKOBmg9DnNBeMaHkBLsClYfcu6eJrbj9y2G7lzcqu+jUZnhJpQoomDU0L1luTMvkapmOCtfzcP/2s3D+3m4STvpJAZrEdeVlDGe+fBSRz40XnQ9Xptv1V/Wz7+eP7Oj71eNwp6Jyd+MJP1vnPit5ZPYUQ4S1FpVT60E8G1FFmG0h5NvGrckVFW4rIUvkJ9iRRetKAJ3/4Heoi8DDOvHQgwz0iCUP4cioyihFRIUCqDS8Yp44NZdAlsp1B/TvkKh3kppGAzsNTD/69FVoxQqezNgw2cjPDYcPX5JHAolD4jenhcPQDIzVvo940j/b5jgvtOglKrh4G1QcvzIk5FwfUjJZZSmTWnBhiZkLMnFSyGSiS0x7O7Y9CywI3aZtJSVls4D0/kd4wPjIXgms4LTeeBFEJDMm9o0KIMIXkuRS6ZeaS0kOUHlwhFTolG2pghL1ZrrxfFnh9Enfcdz438eFq80/a/JtLJ2KVzQ8gAuXYWx0gjx5G1FWbc9sOogiyUdDKRkMxRQyLRSYi9kIsZMWqBpZFIPSdJ8set9oz45wPnimbYE5fsJTnlx5gp/BsKf2dykMEpVKMNpv0K1ayp2uzR8TJlqkhppAcgcSSusQTapksXS91ZaDeHbluR6xBmvbI9TMpqC79qEztWVxPcsCwzNStRS4bUuG1yxVumIRG0NIMLDc1fGr4lTP1tbVyrJYKnbFBI/PhaeYXSYuSZjTbbjXibC6nNQa+qrE1aXMu7ZERjlsYpYVkhEY6acNiE/zTBrF6wk6e+q1DDy9upC5+HyME9jz743chUZ6fXNfI/9Nom2SRjyHW8UA8HpSNLBVL2OwXGtYArvINrkhX6oDLa5RRCP/jUMc0RdLpe58w9ib2Tjtm7oe8FfjRz3oCnIsvEjalDs+AhJ3eZIBRumB6a1JAevQLk5jwmBPdTHgCwzO60jsGq5+mtq94RhUev/DKWhloyPrAOlqKjuxwtOLby9PbRZzlR6kZIOkGr3kwCxrPt9hWs+v1a+w4Oxha0WmCtemgBfFtu0Q0GxW3f7M526wKToYD6/Yb4MTyUXh29AtumaAy42JrLzKqKaEtd01uTMXR7njM3DD/3gvavcVVnMudbWF5aslFZTEoTqU0tEU7NoDN/Wj/nxEazy7AdcyMcwz8B/a8p9M8buxm/Cv1Z+b9AMUljnoUZ3UHXj/ywnNZxuxO06nuUSXMDKvvKjNb5zXxs7c+/+zqsWIxD0/Vev1+DOXauXquYFHq42IMN0+8zuO3qzzCt9hW8gfo6OY/By38+1uUED9B/tO++/wDbJnKwNkeo3697PIbdBryEe0KoV44bM/TLy+LeWm37492nebc9z81TaBb6hL5JAx/vbXS+wil1r8jepZjtNWbYISYS9f7uhTmZFcOrm8DjEd5pmwEefYnDL2Hkn86GN6GpWjeud6Y3tnLGaOTINTBa3oTc9vsQkF8zKfgIuW5sXwgml4ZGTrjuUKjod+H5TKqdbt/WxrW/AgAA///LSOTXfg8AAA==", @@ -682,6 +684,7 @@ func TestK8sVarsMastersOnly(t *testing.T) { "etcIssue": getBase64EncodedGzippedCustomScript(etcIssue), "etcIssueNet": getBase64EncodedGzippedCustomScript(etcIssueNet), "cisNetEnforcement": getBase64EncodedGzippedCustomScript(cisNetEnforcement), + "modprobeConfCIS": getBase64EncodedGzippedCustomScript(modprobeConfCIS), }, "provisionScriptParametersCommon": fmt.Sprintf("[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' CONTAINERD_VERSION=',parameters('containerdVersion'),' MOBY_VERSION=',parameters('mobyVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION=%s HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiServerCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' NETWORK_POLICY=',parameters('networkPolicy'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' MAXIMUM_LOADBALANCER_RULE_COUNT=',variables('maximumLoadBalancerRuleCount'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' IS_HOSTED_MASTER=false',' PRIVATE_AZURE_REGISTRY_SERVER=',parameters('privateAzureRegistryServer'),' AUTHENTICATION_METHOD=',variables('customCloudAuthenticationMethod'),' IDENTITY_SYSTEM=',variables('customCloudIdentifySystem'))]", testK8sVersion), "provisionScriptParametersMaster": "[concat('COSMOS_URI= MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true NO_OUTBOUND=false CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", diff --git a/pkg/engine/const.go b/pkg/engine/const.go index 0a35d3389e..9ec521133c 100644 --- a/pkg/engine/const.go +++ b/pkg/engine/const.go @@ -170,6 +170,7 @@ const ( etcIssue = "k8s/cloud-init/artifacts/etc-issue" etcIssueNet = "k8s/cloud-init/artifacts/etc-issue.net" cisNetEnforcement = "k8s/cloud-init/artifacts/sysctl-d-60-CIS.conf" + modprobeConfCIS = "k8s/cloud-init/artifacts/modprobe-CIS.conf" ) const ( diff --git a/test/e2e/kubernetes/kubernetes_test.go b/test/e2e/kubernetes/kubernetes_test.go index dba306c976..c0da890db1 100644 --- a/test/e2e/kubernetes/kubernetes_test.go +++ b/test/e2e/kubernetes/kubernetes_test.go @@ -236,6 +236,37 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu } }) + It("should validate kernel module configuration", func() { + if eng.ExpandedDefinition.Properties.IsUbuntuDistroForAllNodes() { + kubeConfig, err := GetConfig() + Expect(err).NotTo(HaveOccurred()) + master := fmt.Sprintf("%s@%s", eng.ExpandedDefinition.Properties.LinuxProfile.AdminUsername, kubeConfig.GetServerName()) + nodeList, err := node.Get() + Expect(err).NotTo(HaveOccurred()) + modprobeConfigValidateScript := "modprobe-config-validate.sh" + cmd := exec.Command("scp", "-i", masterSSHPrivateKeyFilepath, "-o", "StrictHostKeyChecking=no", filepath.Join(ScriptsDir, modprobeConfigValidateScript), master+":/tmp/"+modprobeConfigValidateScript) + util.PrintCommand(cmd) + out, err := cmd.CombinedOutput() + log.Printf("%s\n", out) + Expect(err).NotTo(HaveOccurred()) + var conn *remote.Connection + conn, err = remote.NewConnection(kubeConfig.GetServerName(), "22", eng.ExpandedDefinition.Properties.LinuxProfile.AdminUsername, masterSSHPrivateKeyFilepath) + Expect(err).NotTo(HaveOccurred()) + for _, node := range nodeList.Nodes { + err := conn.CopyToRemote(node.Metadata.Name, "/tmp/"+modprobeConfigValidateScript) + Expect(err).NotTo(HaveOccurred()) + netConfigValidationCommand := fmt.Sprintf("\"/tmp/%s\"", modprobeConfigValidateScript) + cmd = exec.Command("ssh", "-A", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR", master, "ssh", "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR", node.Metadata.Name, netConfigValidationCommand) + util.PrintCommand(cmd) + out, err = cmd.CombinedOutput() + log.Printf("%s\n", out) + Expect(err).NotTo(HaveOccurred()) + } + } else { + Skip("cloud-init files validation only works on ubuntu distro until this lands in a VHD") + } + }) + It("should report all nodes in a Ready state", func() { nodeCount := eng.NodeCount() log.Printf("Checking for %d Ready nodes\n", nodeCount) diff --git a/test/e2e/kubernetes/scripts/modprobe-config-validate.sh b/test/e2e/kubernetes/scripts/modprobe-config-validate.sh new file mode 100755 index 0000000000..3a17187d46 --- /dev/null +++ b/test/e2e/kubernetes/scripts/modprobe-config-validate.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x +modprobe -n -v tipc | egrep 'install /bin/true' || exit 1 +[ -s `lsmod | grep tipc` ] || exit 1 +modprobe -n -v dccp | egrep 'install /bin/true' || exit 1 +[ -s `lsmod | grep dccp` ] || exit 1 +modprobe -n -v sctp | egrep 'install /bin/true' || exit 1 +[ -s `lsmod | grep sctp` ] || exit 1 +modprobe -n -v rds | egrep 'install /bin/true' || exit 1 +[ -s `lsmod | grep rds` ] || exit 1 \ No newline at end of file