Merge branch 'main' into cloudbeat-codeowners

tests/commonlib/docker_wrapper.py

@@ -26,5 +26,7 @@ def exec_command(self, container_name: str, command: str, param_value: str, reso
         command_f = f"{command} {param_value} {resource}"
         exit_code, output = container.exec_run(cmd=command_f)
         if exit_code > 0:
-            return ''
+            raise ValueError(f'Failed to execute command: {command_f, output}')
+
         return output.decode().strip()
+

tests/commonlib/io_utils.py

@@ -9,6 +9,7 @@
 import shutil
 from pathlib import Path
 from munch import Munch
+# import grp, pwd
 
 
 def get_logs_from_stream(stream: str) -> list[Munch]:
@@ -25,7 +26,14 @@ def get_logs_from_stream(stream: str) -> list[Munch]:
     for log in logs:
         # current_log = log.split(sep="Z ")[1]
         if log and "bundles" in log:
-            result.append(Munch(json.loads(log)))
+            try:
+                result.append(Munch(json.loads(log)))
+            except json.decoder.JSONDecodeError:
+                result.append(Munch(json.loads(log.replace("'", '"'))))
+            except Exception as e:
+                print(e)
+                continue
+
     return result
 
 
@@ -64,6 +72,34 @@ def exec_command(container_name: str, command: str, param_value: str, resource:
         @return: None
         """
 
+        if command == 'touch':
+            if os.path.exists(param_value):
+                return
+            else:
+                open(param_value, "a+")
+                return
+
+        # if command == 'getent' and param_value == 'group':
+        #     try:
+        #         grp.getgrnam(param_value)
+        #         return ['etcd']
+        #     except KeyError:
+        #         return []
+        #
+        # if command == 'getent' and param_value == 'passwd':
+        #     try:
+        #         pwd.getpwnam(param_value)
+        #         return ['etcd']
+        #     except KeyError:
+        #         return []
+        #
+        # if command == 'groupadd' and param_value == 'etcd':
+        #     try:
+        #         grp.getgrnam(param_value)
+        #         return ['etcd']
+        #     except KeyError:
+        #         return []
+
         if container_name == '':
             raise Exception(f"Unknown {container_name} is sent")
 

tests/configuration.py

@@ -10,7 +10,7 @@
 agent = Munch()
 agent.name = os.getenv('AGENT_NAME', 'cloudbeat')
 agent.namespace = os.getenv('AGENT_NAMESPACE', 'kube-system')
-agent.findings_timeout = 90
+agent.findings_timeout = 30
 
 # --- Kubernetes environment definition --------------------
 kubernetes = Munch()

tests/deploy/cloudbeat-pytest.yml

@@ -143,7 +143,7 @@ data:
   cloudbeat.yml: |-
     cloudbeat:
       # Defines how often an event is sent to the output
-      period: 30s
+      period: 5s
       fetchers:
       - name: kube-api
       - name: process

tests/deploy/k8s-cloudbeat-tests/templates/tests/cloudbeat-test-pod.yml

@@ -24,13 +24,34 @@ spec:
       - name: cloudbeat-test-pv-storage
         mountPath: /usr/src/app/tests/report
       - name: etc-kubernetes
-        mountPath: /etc/kubernetes/manifests
+        mountPath: /etc/kubernetes/
         readOnly: false
+      - name: var-lib-etcd
+        mountPath: /var/lib/etcd
+        readOnly: false
+      - name: kubelet-service
+        mountPath: /etc/systemd/system/kubelet.service.d
+        readOnly: false
+      - name: var-lib-kubelet
+        mountPath: /var/lib/kubelet
+        readOnly: false
+
   restartPolicy: Never
   volumes:
     - name: cloudbeat-test-pv-storage
       hostPath:
         path: /tmp/data
     - name: etc-kubernetes
       hostPath:
-        path: /etc/kubernetes/manifests
+        path: /etc/kubernetes/
+    - name: var-lib-etcd
+      hostPath:
+        path: /var/lib/etcd
+    - name: kubelet-service
+      hostPath:
+        path: /etc/systemd/system/kubelet.service.d
+    - name: var-lib-kubelet
+      hostPath:
+        path: /var/lib/kubelet
+
+

tests/deploy/values/ci.yml

@@ -2,4 +2,4 @@ cloudbeat:
   deploy: true
 
 testData:
-  marker: ci_cloudbeat
+  marker: ci_cloudbeat

tests/product/tests/conftest.py

@@ -0,0 +1,55 @@
+import pytest
+import time
+from commonlib.io_utils import  get_k8s_yaml_objects
+from pathlib import Path
+
+DEPLOY_YAML = "../../deploy/cloudbeat-pytest.yml"
+
+
+@pytest.fixture(scope='module')
+def data(k8s, api_client, cloudbeat_agent):
+    file_path = Path(__file__).parent / DEPLOY_YAML
+    if k8s.get_agent_pod_instances(agent_name=cloudbeat_agent.name, namespace=cloudbeat_agent.namespace):
+        k8s.stop_agent(get_k8s_yaml_objects(file_path=file_path))
+    k8s.start_agent(yaml_file=file_path, namespace=cloudbeat_agent.namespace)
+    time.sleep(5)
+
+    yield k8s, api_client, cloudbeat_agent
+    k8s_yaml_list = get_k8s_yaml_objects(file_path=file_path)
+    k8s.stop_agent(yaml_objects_list=k8s_yaml_list)
+
+
+@pytest.fixture(scope='module')
+def config_node_pre_test(data):
+    k8s_client, api_client, cloudbeat_agent = data
+
+    node = k8s_client.get_cluster_nodes()[0]
+
+    # add etcd group if not exists
+    # groups = api_client.exec_command(container_name=node.metadata.name, command='getent', param_value='group',
+    #                                  resource='')
+    # if 'etcd' not in groups:
+    #     api_client.exec_command(container_name=node.metadata.name, command='groupadd',
+    #                             param_value='etcd',
+    #                             resource='')
+    #
+    # # add etcd user if not exists
+    # users = api_client.exec_command(container_name=node.metadata.name, command='getent', param_value='passwd',
+    #                                 resource='')
+    # if 'etcd' not in users:
+    #     api_client.exec_command(container_name=node.metadata.name,
+    #                             command='useradd',
+    #                             param_value='-g etcd etcd',
+    #                             resource='')
+
+    # create stub file
+    # etcd_content = api_client.exec_command(container_name=node.metadata.name, command='ls',
+    #                                        param_value='/var/lib/etcd/', resource='')
+    # if 'some_file.txt' not in etcd_content:
+
+    api_client.exec_command(container_name=node.metadata.name,
+                            command='touch',
+                            param_value='/var/lib/etcd/some_file.txt',
+                            resource='')
+
+    yield k8s_client, api_client, cloudbeat_agent

tests/product/tests/test_rules.py

@@ -2,45 +2,87 @@
 Kubernetes CIS rules verification.
 This module verifies correctness of retrieved findings by manipulating audit and remediation actions
 """
+import datetime
 import time
 
 import pytest
-from commonlib.io_utils import get_logs_from_stream, get_k8s_yaml_objects
-from pathlib import Path
+from commonlib.io_utils import get_logs_from_stream
+from product.tests.tests.file_system.file_system_test_cases import *
 
 
-DEPLOY_YAML = "../../deploy/cloudbeat-pytest.yml"
-
-
-@pytest.fixture(scope='module')
-def data(k8s, api_client, cloudbeat_agent):
+def check_logs(k8s, timeout, pod_name, namespace, rule_tag, expected, exec_timestamp) -> bool:
+    """
+    This function retrieves pod logs and verifies if evaluation result is equal to expected result.
+    @param k8s: Kubernetes wrapper instance
+    @param timeout: Exit timeout
+    @param pod_name: Name of pod the logs shall be retrieved from
+    @param namespace: Kubernetes namespace
+    @param rule_tag: Log rule tag
+    @param expected: Expected result
+    @:param exec_timestamp: the timestamp the command executed
+    @return: bool True / False
+    """
+    start_time = time.time()
+    iteration = 0
+    while time.time() - start_time < timeout:
+        try:
+            logs = get_logs_from_stream(k8s.get_pod_logs(pod_name=pod_name,
+                                                         namespace=namespace,
+                                                         since_seconds=2))
+        except:
+            continue
+        for log in logs:
+            if not log.get('result'):
+                continue
+            findings = log.get('result').get('findings')
+            findings_timestamp = datetime.datetime.strptime(log["time"], '%Y-%m-%dT%H:%M:%Sz')
+            if (findings_timestamp - exec_timestamp).total_seconds() < 0:
+                continue
 
-    file_path = Path(__file__).parent / DEPLOY_YAML
-    if k8s.get_agent_pod_instances(agent_name=cloudbeat_agent.name, namespace=cloudbeat_agent.namespace):
-        k8s.stop_agent(get_k8s_yaml_objects(file_path=file_path))
-    k8s.start_agent(yaml_file=file_path, namespace=cloudbeat_agent.namespace)
-    time.sleep(5)
-    yield k8s, api_client, cloudbeat_agent
-    k8s_yaml_list = get_k8s_yaml_objects(file_path=file_path)
-    k8s.stop_agent(yaml_objects_list=k8s_yaml_list)
+            if findings:
+                for finding in findings:
+                    if rule_tag in finding.get('rule').get('tags'):
+                        iteration += 1
+                        agent_evaluation = finding.get('result').get('evaluation')
+                        if agent_evaluation == expected:
+                            print(f"{iteration}: expected:"
+                                  f"{expected} tags:"
+                                  f"{finding.get('rule').get('tags')}, "
+                                  f"evidence: {finding.get('result').get('evidence')} ",
+                                  f"evaluation: {finding.get('result').get('evaluation')}")
+                            return True
+    if iteration == 0:
+        raise EnvironmentError("no logs found")
+    return False
 
 
 @pytest.mark.rules
 @pytest.mark.parametrize(
     ("rule_tag", "command", "param_value", "resource", "expected"),
-    [
-        ('CIS 1.1.1', 'chmod', '700', '/etc/kubernetes/manifests/kube-apiserver.yaml', 'failed'),
-        ('CIS 1.1.1', 'chmod', '644', '/etc/kubernetes/manifests/kube-apiserver.yaml', 'passed'),
-        ('CIS 1.1.2', 'chown', 'daemon:daemon', '/etc/kubernetes/manifests/kube-apiserver.yaml', 'failed'),
-        ('CIS 1.1.2', 'chown', 'root:root', '/etc/kubernetes/manifests/kube-apiserver.yaml', 'passed')
-    ],
-    ids=['CIS 1.1.1 mode 700',
-         'CIS 1.1.1 mode 644',
-         'CIS 1.1.2 daemon:daemon',
-         'CIS 1.1.2 root:root'
-         ]
+    [*cis_1_1_1,
+     *cis_1_1_2,
+     *cis_1_1_3,
+     *cis_1_1_4,
+     *cis_1_1_5,
+     *cis_1_1_6,
+     *cis_1_1_7,
+     *cis_1_1_8,
+     *cis_1_1_11,
+     # *cis_1_1_12, uncomment after fix https://github.com/elastic/cloudbeat/issues/118
+     *cis_1_1_13,
+     *cis_1_1_14,
+     *cis_1_1_15,
+     *cis_1_1_16,
+     *cis_1_1_17,
+     *cis_1_1_18,
+     *cis_4_1_1,
+     *cis_4_1_2,
+     *cis_4_1_5,
+     *cis_4_1_9,
+     *cis_4_1_10
+     ],
 )
-def test_master_node_configuration(data,
+def test_file_system_configuration(config_node_pre_test,
                                    rule_tag,
                                    command,
                                    param_value,
@@ -60,56 +102,24 @@ def test_master_node_configuration(data,
     @param expected: Result to be found in finding evaluation field.
     @return: None - Test Pass / Fail result is generated.
     """
-    k8s_client, api_client, agent_config = data
+    k8s_client, api_client, cloudbeat_agent = config_node_pre_test
     # Currently, single node is used, in the future may be extended for all nodes.
     node = k8s_client.get_cluster_nodes()[0]
-    pods = k8s_client.get_agent_pod_instances(agent_name=agent_config.name, namespace=agent_config.namespace)
+    pods = k8s_client.get_agent_pod_instances(agent_name=cloudbeat_agent.name, namespace=cloudbeat_agent.namespace)
+
     api_client.exec_command(container_name=node.metadata.name,
                             command=command,
                             param_value=param_value,
                             resource=resource)
 
+    exec_ts = datetime.datetime.utcnow()
+
     verification_result = check_logs(k8s=k8s_client,
                                      pod_name=pods[0].metadata.name,
-                                     namespace=agent_config.namespace,
+                                     namespace=cloudbeat_agent.namespace,
                                      rule_tag=rule_tag,
                                      expected=expected,
-                                     timeout=agent_config.findings_timeout)
+                                     timeout=cloudbeat_agent.findings_timeout,
+                                     exec_timestamp=exec_ts)
 
     assert verification_result, f"Rule {rule_tag} verification failed."
-
-
-def check_logs(k8s, timeout, pod_name, namespace, rule_tag, expected) -> bool:
-    """
-    This function retrieves pod logs and verifies if evaluation result is equal to expected result.
-    @param k8s: Kubernetes wrapper instance
-    @param timeout: Exit timeout
-    @param pod_name: Name of pod the logs shall be retrieved from
-    @param namespace: Kubernetes namespace
-    @param rule_tag: Log rule tag
-    @param expected: Expected result
-    @return: bool True / False
-    """
-    start_time = time.time()
-    iteration = 0
-    while time.time() - start_time < timeout:
-        logs = get_logs_from_stream(k8s.get_pod_logs(pod_name=pod_name,
-                                                     namespace=namespace,
-                                                     since_seconds=1))
-        iteration += 1
-        for log in logs:
-            if not log.get('result'):
-                print(f"{iteration}: no result")
-                continue
-            findings = log.get('result').get('findings')
-            if findings:
-                for finding in findings:
-                    if rule_tag in finding.get('rule').get('tags'):
-                        print(f"{iteration}: expected:"
-                              f"{expected} tags:"
-                              f"{finding.get('rule').get('tags')}, "
-                              f"evaluation: {finding.get('result').get('evaluation')}")
-                        agent_evaluation = finding.get('result').get('evaluation')
-                        if agent_evaluation == expected:
-                            return True
-    return False