-
Notifications
You must be signed in to change notification settings - Fork 43
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add kube fetcher tests #32
Conversation
Cloudbeat CI 🤖The integration test was completed successfully! 🏆 Exmaple findings from ElasticSearch:file-system 💚{
"@timestamp": "2022-03-29T15:42:02.128Z",
"result": {
"evaluation": "passed",
"evidence": {
"filemode": "644"
}
},
"ecs": {
"version": "8.0.0"
},
"cluster_id": "6b9a87bf-4c90-435f-bc3d-e63cd53952f5",
"rule": {
"impact": "None",
"name": "Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive",
"remediation": "chmod 644 /var/lib/kubelet/config.yaml",
"tags": [
"CIS",
"Kubernetes",
"CIS 4.1.9",
"Worker Node Configuration"
],
"benchmark": {
"name": "CIS Kubernetes V1.20",
"version": "v1.0.0"
},
"description": "Ensure that if the kubelet refers to a configuration file with the --config argument, that file has permissions of 644 or more restrictive."
},
"host": {
"mac": [
"02:42:9e:87:01:1f",
"f6:fa:9c:22:37:1a",
"02:42:c0:a8:31:02"
],
"hostname": "minikube",
"architecture": "x86_64",
"os": {
"type": "linux",
"platform": "debian",
"version": "11 (bullseye)",
"family": "debian",
"name": "Debian GNU/Linux",
"kernel": "5.11.0-1028-azure",
"codename": "bullseye"
},
"name": "minikube",
"containerized": true,
"ip": [
"172.17.0.1",
"192.168.49.2"
]
},
"agent": {
"name": "minikube",
"type": "cloudbeat",
"version": "8.2.0",
"ephemeral_id": "82e20511-d0fb-40af-bfe3-6f49215baf94",
"id": "e32dbf93-87e6-499c-9640-6f32eb41e391"
},
"cloud": {
"machine": {
"type": "Standard_DS2_v2"
},
"service": {
"name": "Virtual Machines"
},
"region": "southcentralus",
"provider": "azure",
"account": {},
"instance": {
"name": "SCUS-GHEUS21U20SCUSC34-0207",
"id": "df9c1df1-755e-4a82-bc84-450cfecc01e9"
}
},
"resource_id": "1594339",
"type": "file-system",
"cycle_id": "6d66f8ec-1754-414f-a748-068f4ea41936",
"resource": {
"inode": "1594339",
"filename": "config.yaml",
"mode": "644",
"gid": "root",
"uid": "root",
"path": "/hostfs/var/lib/kubelet/config.yaml"
}
} process 💚{
"@timestamp": "2022-03-29T15:42:01.464Z",
"host": {
"containerized": true,
"ip": [
"172.17.0.1",
"192.168.49.2"
],
"mac": [
"02:42:9e:87:01:1f",
"f6:fa:9c:22:37:1a",
"02:42:c0:a8:31:02"
],
"hostname": "minikube",
"architecture": "x86_64",
"name": "minikube",
"os": {
"family": "debian",
"name": "Debian GNU/Linux",
"kernel": "5.11.0-1028-azure",
"codename": "bullseye",
"type": "linux",
"platform": "debian",
"version": "11 (bullseye)"
}
},
"agent": {
"ephemeral_id": "82e20511-d0fb-40af-bfe3-6f49215baf94",
"id": "e32dbf93-87e6-499c-9640-6f32eb41e391",
"name": "minikube",
"type": "cloudbeat",
"version": "8.2.0"
},
"cluster_id": "6b9a87bf-4c90-435f-bc3d-e63cd53952f5",
"resource_id": "1743",
"result": {
"evaluation": "failed",
"evidence": {
"process_config": {
"config": {
"cgroupDriver": "cgroupfs",
"clusterDomain": "cluster.local",
"failSwapOn": false,
"healthzBindAddress": "127.0.0.1",
"nodeStatusUpdateFrequency": "0s",
"rotateCertificates": true,
"apiVersion": "kubelet.config.k8s.io/v1beta1",
"authorization": {
"mode": "Webhook",
"webhook": {
"cacheAuthorizedTTL": "0s",
"cacheUnauthorizedTTL": "0s"
}
},
"shutdownGracePeriod": "0s",
"evictionPressureTransitionPeriod": "0s",
"healthzPort": 10248,
"logging": {
"flushFrequency": 0,
"options": {
"json": {
"infoBufferSize": "0"
}
},
"verbosity": 0
},
"cpuManagerReconcilePeriod": "0s",
"evictionHard": {
"imagefs.available": "0%",
"nodefs.available": "0%",
"nodefs.inodesFree": "0%"
},
"httpCheckFrequency": "0s",
"shutdownGracePeriodCriticalPods": "0s",
"runtimeRequestTimeout": "0s",
"syncFrequency": "0s",
"volumeStatsAggPeriod": "0s",
"clusterDNS": [
"10.96.0.10"
],
"kind": "KubeletConfiguration",
"staticPodPath": "/etc/kubernetes/manifests",
"imageMinimumGCAge": "0s",
"memorySwap": {},
"streamingConnectionIdleTimeout": "0s",
"imageGCHighThresholdPercent": 100,
"nodeStatusReportFrequency": "0s",
"authentication": {
"anonymous": {
"enabled": false
},
"webhook": {
"enabled": true,
"cacheTTL": "0s"
},
"x509": {
"clientCAFile": "/var/lib/minikube/certs/ca.crt"
}
},
"fileCheckFrequency": "0s"
}
},
"process_args": {
"--bootstrap-kubeconfig": "/etc/kubernetes/bootstrap-kubelet.conf",
"--config": "/var/lib/kubelet/config.yaml",
"--container-runtime": "docker",
"--hostname-override": "minikube",
"--housekeeping-interval": "5m",
"--kubeconfig": "/etc/kubernetes/kubelet.conf",
"--node-ip": "192.168.49.2",
"/var/lib/minikube/binaries/v1.23.3/kubelet": "/var/lib/minikube/binaries/v1.23.3/kubelet"
}
}
},
"resource": {
"command": "/var/lib/minikube/binaries/v1.23.3/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --config=/var/lib/kubelet/config.yaml --container-runtime=docker --hostname-override=minikube --housekeeping-interval=5m --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.49.2",
"stat": {
"UserTime": "330",
"EffectiveUID": "",
"SavedUID": "",
"Parent": "1",
"Nice": "0",
"RealGID": "",
"EffectiveGID": "",
"State": "S",
"StartTime": "24960",
"SystemTime": "121",
"RealUID": "",
"SavedGID": "",
"ResidentSize": "96184000",
"Group": "1743",
"Name": "kubelet",
"TotalSize": "1977752000",
"Threads": "15"
},
"external_data": {
"config": {
"cgroupDriver": "cgroupfs",
"clusterDNS": [
"10.96.0.10"
],
"clusterDomain": "cluster.local",
"fileCheckFrequency": "0s",
"nodeStatusReportFrequency": "0s",
"runtimeRequestTimeout": "0s",
"authorization": {
"mode": "Webhook",
"webhook": {
"cacheAuthorizedTTL": "0s",
"cacheUnauthorizedTTL": "0s"
}
},
"healthzBindAddress": "127.0.0.1",
"memorySwap": {},
"evictionHard": {
"imagefs.available": "0%",
"nodefs.available": "0%",
"nodefs.inodesFree": "0%"
},
"evictionPressureTransitionPeriod": "0s",
"staticPodPath": "/etc/kubernetes/manifests",
"apiVersion": "kubelet.config.k8s.io/v1beta1",
"failSwapOn": false,
"healthzPort": 10248,
"httpCheckFrequency": "0s",
"nodeStatusUpdateFrequency": "0s",
"shutdownGracePeriodCriticalPods": "0s",
"authentication": {
"x509": {
"clientCAFile": "/var/lib/minikube/certs/ca.crt"
},
"anonymous": {
"enabled": false
},
"webhook": {
"enabled": true,
"cacheTTL": "0s"
}
},
"cpuManagerReconcilePeriod": "0s",
"imageGCHighThresholdPercent": 100,
"imageMinimumGCAge": "0s",
"logging": {
"flushFrequency": 0,
"options": {
"json": {
"infoBufferSize": "0"
}
},
"verbosity": 0
},
"volumeStatsAggPeriod": "0s",
"kind": "KubeletConfiguration",
"shutdownGracePeriod": "0s",
"streamingConnectionIdleTimeout": "0s",
"syncFrequency": "0s",
"rotateCertificates": true
}
},
"pid": "1743"
},
"ecs": {
"version": "8.0.0"
},
"cloud": {
"region": "southcentralus",
"provider": "azure",
"account": {},
"instance": {
"id": "df9c1df1-755e-4a82-bc84-450cfecc01e9",
"name": "SCUS-GHEUS21U20SCUSC34-0207"
},
"machine": {
"type": "Standard_DS2_v2"
},
"service": {
"name": "Virtual Machines"
}
},
"type": "process",
"cycle_id": "6d66f8ec-1754-414f-a748-068f4ea41936",
"rule": {
"impact": "Kubelet would manage the iptables on the system and keep it in sync. If you are using any other iptables management solution, then there might be some conflicts.",
"name": "Ensure that the --make-iptables-util-chains argument is set to true",
"remediation": "If using a Kubelet config file, edit the file to set makeIPTablesUtilChains: true. If using command line arguments, edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and remove the --make-iptables-util-chains argument from the KUBELET_SYSTEM_PODS_ARGS variable. Based on your system, restart the kubelet service.",
"tags": [
"CIS",
"Kubernetes",
"CIS 4.2.7",
"Kubelet"
],
"benchmark": {
"name": "CIS Kubernetes V1.20",
"version": "v1.0.0"
},
"description": "Allow Kubelet to manage iptables."
}
} kube-api 💚{
"@timestamp": "2022-03-29T15:42:01.936Z",
"resource": {
"kind": "ClusterRole",
"apiVersion": "rbac.authorization.k8s.io/v1",
"metadata": {
"name": "system:controller:horizontal-pod-autoscaler",
"uid": "710d22e7-052a-4eb4-9235-6f3b861441a0",
"resourceVersion": "129",
"creationTimestamp": "2022-03-29T15:37:36Z",
"labels": {
"kubernetes.io/bootstrapping": "rbac-defaults"
},
"annotations": {
"rbac.authorization.kubernetes.io/autoupdate": "true"
}
},
"rules": [
{
"verbs": [
"get",
"list",
"watch"
],
"apiGroups": [
"autoscaling"
],
"resources": [
"horizontalpodautoscalers"
]
},
{
"verbs": [
"update"
],
"apiGroups": [
"autoscaling"
],
"resources": [
"horizontalpodautoscalers/status"
]
},
{
"verbs": [
"get",
"update"
],
"apiGroups": [
"*"
],
"resources": [
"*/scale"
]
},
{
"verbs": [
"list"
],
"apiGroups": [
""
],
"resources": [
"pods"
]
},
{
"verbs": [
"get"
],
"apiGroups": [
""
],
"resources": [
"services/proxy"
],
"resourceNames": [
"http:heapster:",
"https:heapster:"
]
},
{
"resources": [
"pods"
],
"verbs": [
"list"
],
"apiGroups": [
"metrics.k8s.io"
]
},
{
"verbs": [
"get",
"list"
],
"apiGroups": [
"custom.metrics.k8s.io"
],
"resources": [
"*"
]
},
{
"verbs": [
"create",
"patch",
"update"
],
"apiGroups": [
"",
"events.k8s.io"
],
"resources": [
"events"
]
}
]
},
"rule": {
"benchmark": {
"name": "CIS Kubernetes V1.20",
"version": "v1.0.0"
},
"description": "Kubernetes Roles and ClusterRoles provide access to resources based on sets of objects and actions that can be taken on those objects.\nIt is possible to set either of these to be the wildcard \"*\" which matches all items.\nUse of wildcards is not optimal from a security perspective as it may allow for inadvertent access to be granted when new resources are added to the Kubernetes API either as CRDs or in later versions of the product.",
"impact": "None",
"name": "Minimize wildcard use in Roles and ClusterRoles",
"remediation": "Where possible replace any use of wildcards in clusterroles and roles with specific objects or actions.",
"tags": [
"CIS",
"Kubernetes",
"CIS 5.1.3",
"RBAC and Service Accounts"
]
},
"type": "kube-api",
"host": {
"ip": [
"172.17.0.1",
"192.168.49.2"
],
"mac": [
"02:42:9e:87:01:1f",
"f6:fa:9c:22:37:1a",
"02:42:c0:a8:31:02"
],
"name": "minikube",
"hostname": "minikube",
"architecture": "x86_64",
"os": {
"name": "Debian GNU/Linux",
"kernel": "5.11.0-1028-azure",
"codename": "bullseye",
"type": "linux",
"platform": "debian",
"version": "11 (bullseye)",
"family": "debian"
},
"containerized": true
},
"agent": {
"type": "cloudbeat",
"version": "8.2.0",
"ephemeral_id": "82e20511-d0fb-40af-bfe3-6f49215baf94",
"id": "e32dbf93-87e6-499c-9640-6f32eb41e391",
"name": "minikube"
},
"cluster_id": "6b9a87bf-4c90-435f-bc3d-e63cd53952f5",
"resource_id": "710d22e7-052a-4eb4-9235-6f3b861441a0",
"cycle_id": "6d66f8ec-1754-414f-a748-068f4ea41936",
"result": {
"evaluation": "failed",
"evidence": {
"cluster_roles": {
"apiVersion": "rbac.authorization.k8s.io/v1",
"kind": "ClusterRole",
"metadata": {
"creationTimestamp": "2022-03-29T15:37:36Z",
"labels": {
"kubernetes.io/bootstrapping": "rbac-defaults"
},
"name": "system:controller:horizontal-pod-autoscaler",
"resourceVersion": "129",
"uid": "710d22e7-052a-4eb4-9235-6f3b861441a0",
"annotations": {
"rbac.authorization.kubernetes.io/autoupdate": "true"
}
},
"rules": [
{
"apiGroups": [
"autoscaling"
],
"resources": [
"horizontalpodautoscalers"
],
"verbs": [
"get",
"list",
"watch"
]
},
{
"apiGroups": [
"autoscaling"
],
"resources": [
"horizontalpodautoscalers/status"
],
"verbs": [
"update"
]
},
{
"apiGroups": [
"*"
],
"resources": [
"*/scale"
],
"verbs": [
"get",
"update"
]
},
{
"apiGroups": [
""
],
"resources": [
"pods"
],
"verbs": [
"list"
]
},
{
"apiGroups": [
""
],
"resourceNames": [
"http:heapster:",
"https:heapster:"
],
"resources": [
"services/proxy"
],
"verbs": [
"get"
]
},
{
"apiGroups": [
"metrics.k8s.io"
],
"resources": [
"pods"
],
"verbs": [
"list"
]
},
{
"apiGroups": [
"custom.metrics.k8s.io"
],
"resources": [
"*"
],
"verbs": [
"get",
"list"
]
},
{
"apiGroups": [
"",
"events.k8s.io"
],
"resources": [
"events"
],
"verbs": [
"create",
"patch",
"update"
]
}
]
}
}
},
"ecs": {
"version": "8.0.0"
},
"cloud": {
"instance": {
"name": "SCUS-GHEUS21U20SCUSC34-0207",
"id": "df9c1df1-755e-4a82-bc84-450cfecc01e9"
},
"machine": {
"type": "Standard_DS2_v2"
},
"service": {
"name": "Virtual Machines"
},
"region": "southcentralus",
"provider": "azure",
"account": {}
}
} |
Cloudbeat CI 🤖Integration tests status: success =========================== short test summary info ============================
PASSED integration/tests/test_output_to_elasticsearch.py::test_elastic_index_exists[file-system]
PASSED integration/tests/test_output_to_elasticsearch.py::test_elastic_index_exists[process]
PASSED integration/tests/test_output_to_elasticsearch.py::test_elastic_index_exists[kube-api]
PASSED product/tests/test_cloudbeat.py::test_cloudbeat_pod_exist
PASSED product/tests/test_cloudbeat.py::test_cloudbeat_pods_running
================= 5 passed, 4 deselected, 6 warnings in 1.40s ================== Link to detailed report: https://elastic.github.io/cloudbeat/411 |
This pull request is now in conflicts. Could you fix it? 🙏
|
This pull request does not have a backport label. Could you fix it @eyalkraft? 🙏
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thank you for adding those tests 💪
Few minor comments...
resources/fetchers/kube_factory.go
Outdated
} | ||
|
||
func (f *KubeFactory) CreateFrom(log *logp.Logger, cfg KubeApiFetcherConfig, ch chan fetching.ResourceInfo) (fetching.Fetcher, error) { | ||
func (f *KubeFactory) CreateFrom(log *logp.Logger, cfg KubeApiFetcherConfig, ch chan fetching.ResourceInfo, provider func(kubeconfig string, opt kubernetes.KubeClientOptions) (k8s.Interface, error)) (fetching.Fetcher, error) { |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It would be nice if you could create a type for this func, will be more readable.
Something like: type K8sProviderFunc func(string, kubernetes.KubeClientOptions) (k8s.Interface, error)
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done
f := reflect.Indirect(r).FieldByName("Items") | ||
items := f.Interface() | ||
// Finding a way to avoid this switch case could be nice | ||
switch items := items.(type) { |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Might do the trick
switch items := items.(type) { | |
return PtrMap(items.([]any) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Doesn't work.
panic: interface conversion: interface {} is []v1.Pod, not []interface {} [recovered]
for i, tt := range tests { | ||
s.Run(fmt.Sprintf("Kube api test %v", i), func() { | ||
client := k8sfake.NewSimpleClientset(tt) | ||
provider := MockProvider(client) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can be part of the suite if you want to.
can utilize SetupTest and TearDownTest for this purpose.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I use tt
so seems not possible
|
||
func (s *KubeFetcherTestSuite) TestKubeFetcher_TestFetch() { | ||
empty := v1.PodList{} | ||
pod := v1.PodList{Items: []v1.Pod{{ |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we reuse pod in pods?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
done
close(s.resourceCh) | ||
} | ||
|
||
func MockProvider(client *k8sfake.Clientset) func(s string, options kubernetes.KubeClientOptions) (k8s.Interface, error) { |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can it be used here as well?
func MockProvider(client *k8sfake.Clientset) func(s string, options kubernetes.KubeClientOptions) (k8s.Interface, error) { | |
func MockProvider(client *k8sfake.Clientset) KubeClientProvider { |
Relates https://github.com/elastic/security-team/issues/3130