feat: New ActivityAuditClientV1 to query Activity Audit events (#197)

* feat: New ActivityAuditClientV1 to query Activity Audit events
* chore: Deprecate old command audit methods
* refactor: Import datetime package directly
* fix(lint): Solve linting problems
* fix(ci): Change the image to scan to quay.io/sysdig/agent

sdcclient/_secure.py

@@ -496,6 +496,8 @@ def list_commands_audit(self, from_sec=None, to_sec=None, scope_filter=None, com
         '''**Description**
             List the commands audit.
 
+        **DEPRECATED**: Use sdcclient.secure.ActivityAuditClientV1 instead. This is maintained for old on-prem versions, but will be removed over time.
+
         **Arguments**
             - from_sec: the start of the timerange for which to get commands audit.
             - end_sec: the end of the timerange for which to get commands audit.
@@ -528,6 +530,8 @@ def get_command_audit(self, id, metrics=[]):
         '''**Description**
             Get a command audit.
 
+        **DEPRECATED**: Use sdcclient.secure.ActivityAuditClientV1 instead. This is maintained for old on-prem versions, but will be removed over time.
+
         **Arguments**
             - id: the id of the command audit to get.
 

sdcclient/secure/__init__.py

@@ -1,8 +1,10 @@
+from ._activity_audit_v1 import ActivityAuditClientV1, ActivityAuditDataSource
 from ._falco_rules_files_old import FalcoRulesFilesClientOld
 from ._policy_events_old import PolicyEventsClientOld
 from ._policy_events_v1 import PolicyEventsClientV1
-from ._policy_v2 import PolicyClientV2, policy_action_pause, policy_action_stop, policy_action_kill, \
-    policy_action_capture
+from ._policy_v2 import policy_action_capture, policy_action_kill, policy_action_pause, policy_action_stop, \
+    PolicyClientV2
 
 __all__ = ["PolicyEventsClientOld", "PolicyEventsClientV1", "FalcoRulesFilesClientOld",
-           "PolicyClientV2", "policy_action_pause", "policy_action_stop", "policy_action_kill", "policy_action_capture"]
+           "PolicyClientV2", "policy_action_pause", "policy_action_stop", "policy_action_kill", "policy_action_capture",
+           "ActivityAuditClientV1", "ActivityAuditDataSource"]

sdcclient/secure/_activity_audit_v1.py

@@ -0,0 +1,137 @@
+import datetime
+
+from sdcclient._common import _SdcCommon
+
+
+class ActivityAuditDataSource:
+    CMD = "command"
+    NET = "connection"
+    KUBE_EXEC = "kubernetes"
+    FILE = "fileaccess"
+
+
+_seconds_to_nanoseconds = 10 ** 9
+
+
+class ActivityAuditClientV1(_SdcCommon):
+    def __init__(self, token="", sdc_url='https://secure.sysdig.com', ssl_verify=True, custom_headers=None):
+        super(ActivityAuditClientV1, self).__init__(token, sdc_url, ssl_verify, custom_headers)
+        self.product = "SDS"
+
+    def list_events(self, from_date=None, to_date=None, scope_filter=None, limit=0,
+                    data_sources=None):
+        """
+        List the events in the Activity Audit.
+
+        Args:
+            from_date (datetime.datetime): the start of the time range from which to get events. The default value is yesterday.
+            to_date (datetime.datetime): the end of the time range from which to get events. The default value is now.
+            scope_filter (List): a list of Sysdig Monitor-like filter (e.g `processName in ("ubuntu")`).
+            limit (int): max number of events to retrieve. A limit of 0 or negative will retrieve all events.
+            data_sources (List): a list of data sources to retrieve events from. None or an empty list retrieves all events.
+
+        Examples:
+            >>> client = ActivityAuditClientV1(token=SECURE_TOKEN)
+            >>>
+            >>> now = datetime.datetime.utcnow()
+            >>> three_days_ago = now - datetime.timedelta(days=3)
+            >>> max_event_number_retrieved = 50
+            >>> data_sources = [ActivityAuditDataSource.CMD, ActivityAuditDataSource.KUBE_EXEC]
+            >>>
+            >>> ok, events = client.list_events(from_date=three_days_ago,
+            >>>                                 to_date=now,
+            >>>                                 limit=max_event_number_retrieved,
+            >>>                                 data_sources=data_sources)
+
+        Returns:
+            A list of event objects from the Activity Audit.
+        """
+        number_of_events_per_query = 50
+
+        if from_date is None:
+            from_date = datetime.datetime.utcnow() - datetime.timedelta(days=1)
+        if to_date is None:
+            to_date = datetime.datetime.utcnow()
+
+        filters = scope_filter if scope_filter else []
+        if data_sources:
+            quoted_data_sources = [f'"{data_source}"' for data_source in data_sources]
+            data_source_filter = f'type in ({",".join(quoted_data_sources)})'
+            filters.append(data_source_filter)
+
+        query_params = {
+            "from": int(from_date.timestamp()) * _seconds_to_nanoseconds,
+            "to": int(to_date.timestamp()) * _seconds_to_nanoseconds,
+            "limit": number_of_events_per_query,
+            "filter": " and ".join(filters),
+        }
+
+        res = self.http.get(self.url + '/api/v1/activityAudit/events', headers=self.hdrs, verify=self.ssl_verify,
+                            params=query_params)
+        ok, res = self._request_result(res)
+        if not ok:
+            return False, res
+
+        events = []
+
+        # Pagination required by Secure API
+        while "page" in res and \
+                "total" in res["page"] and \
+                res["page"]["total"] > number_of_events_per_query:
+            events = events + res["data"]
+
+            if 0 < limit < len(events):
+                events = events[0:limit - 1]
+                break
+
+            paginated_query_params = {
+                "limit": number_of_events_per_query,
+                "filter": " and ".join(filters),
+                "cursor": res["page"]["prev"]
+            }
+
+            res = self.http.get(self.url + '/api/v1/activityAudit/events', headers=self.hdrs, verify=self.ssl_verify,
+                                params=paginated_query_params)
+            ok, res = self._request_result(res)
+            if not ok:
+                return False, res
+        else:
+            events = events + res["data"]
+
+        return True, events
+
+    def list_trace(self, traceable_event):
+        """
+        Lists the events from an original traceable event.
+
+        Args:
+            traceable_event(object): an event retrieved from the list_events method. The event must be traceable,
+                                     this is, it must have the "traceable" key as true.
+
+        Examples:
+            >>> client = ActivityAuditClientV1(token=SECURE_TOKEN)
+            >>>
+            >>> ok, events = client.list_events()
+            >>> if not ok:
+            >>>     return
+            >>> traceable_events = [event for event in events if event["traceable"]]
+            >>>
+            >>> ok, trace = client.list_trace(traceable_events[0])
+            >>> if not ok:
+            >>>     return
+            >>>
+            >>> for event in trace:
+            >>>     print(event)
+
+        Returns:
+            All the related events that are the trace of the given event.
+        """
+        if not traceable_event or not traceable_event["traceable"]:
+            return False, "a traceable event must be provided"
+
+        endpoint = f'/api/v1/activityAudit/events/{traceable_event["type"]}/{traceable_event["id"]}/trace'
+        res = self.http.get(self.url + endpoint, headers=self.hdrs, verify=self.ssl_verify)
+        ok, res = self._request_result(res)
+        if not ok:
+            return False, res
+        return True, res["data"]

specs/secure/activitylog_v1_spec.py

@@ -0,0 +1,58 @@
+import datetime
+import os
+
+from expects import be_above, be_empty, contain, expect, have_keys, have_len
+from mamba import _it, before, context, description, it
+
+from sdcclient.secure import ActivityAuditClientV1 as ActivityAuditClient, ActivityAuditDataSource
+from specs import be_successful_api_call
+
+with description("Activity Audit v1", "integration") as self:
+    with before.all:
+        self.client = ActivityAuditClient(sdc_url=os.getenv("SDC_SECURE_URL", "https://secure.sysdig.com"),
+                                          token=os.getenv("SDC_SECURE_TOKEN"))
+
+    with it("is able to list the most recent commands with the default parameters"):
+        ok, res = self.client.list_events()
+
+        expect((ok, res)).to(be_successful_api_call)
+        expect(res).to_not(be_empty)
+
+    with context("when listing the most recent commands with a limit of 5"):
+        with it("retrieves the 5 events"):
+            ok, res = self.client.list_events(limit=5)
+
+            expect((ok, res)).to(be_successful_api_call)
+            expect(res).to_not(have_len(5))
+
+    with context("when listing the events from the last 3 days"):
+        with it("retrieves all the events"):
+            three_days_ago = datetime.datetime.utcnow() - datetime.timedelta(days=3)
+            ok, res = self.client.list_events(from_date=three_days_ago)
+
+            expect((ok, res)).to(be_successful_api_call)
+            expect(res).to_not(be_empty)
+
+    with context("when listing events from a specific type"):
+        with it("retrieves the events of this event type only"):
+            ok, res = self.client.list_events(data_sources=[ActivityAuditDataSource.CMD])
+
+            expect((ok, res)).to(be_successful_api_call)
+            expect(res).to(contain(have_keys(type=ActivityAuditDataSource.CMD)))
+            expect(res).to_not(contain(have_keys(type=ActivityAuditDataSource.KUBE_EXEC)))
+            expect(res).to_not(contain(have_keys(type=ActivityAuditDataSource.FILE)))
+            expect(res).to_not(contain(have_keys(type=ActivityAuditDataSource.NET)))
+
+    with context("when retrieving the inner events of a traceable event"):
+        with _it("retrieves the trace of these events"):
+            ok, res = self.client.list_events(data_sources=[ActivityAuditDataSource.KUBE_EXEC])
+            expect((ok, res)).to(be_successful_api_call)
+
+            expect(res).to(contain(have_keys(traceable=True)))
+
+            traceable_events = [event for event in res if event["traceable"]]
+            ok, res = self.client.list_trace(traceable_events[0])
+
+            expect((ok, res)).to(be_successful_api_call)
+            expect(res).to(contain(have_keys(type=ActivityAuditDataSource.CMD)))
+            expect(res).to(have_len(be_above(0)))  # Not using be_empty, because we want to ensure this is a list

specs/secure/scanning/policy_evaluation_spec.py

@@ -11,50 +11,47 @@
     with before.all:
         self.client = SdScanningClient(sdc_url=os.getenv("SDC_SECURE_URL", "https://secure.sysdig.com"),
                                        token=os.getenv("SDC_SECURE_TOKEN"))
+        self.image_name = "quay.io/sysdig/agent:latest"
 
     with it("is able to retrieve the results for all the policies"):
-        image_name = "alpine:latest"
-        ok, res = self.client.get_image_scanning_results(image_name)
+        ok, res = self.client.get_image_scanning_results(self.image_name)
 
         expect((ok, res)).to(be_successful_api_call)
         expect(res).to(
             have_keys("image_digest", "image_id", "stop_results",
                       total_warn=be_above_or_equal(0), total_stop=be_above_or_equal(0),
                       last_evaluation=be_an(datetime),
-                      status="pass", image_tag="docker.io/alpine:latest",
+                      status="pass", image_tag=self.image_name,
                       policy_id="*", policy_name="All policies",
                       warn_results=not_(be_empty))
         )
 
     with it("is able to retrieve the results for the default policy"):
-        image_name = "alpine:latest"
         policy_id = "default"
-        ok, res = self.client.get_image_scanning_results(image_name, policy_id)
+        ok, res = self.client.get_image_scanning_results(self.image_name, policy_id)
 
         expect((ok, res)).to(be_successful_api_call)
         expect(res).to(
             have_keys("image_digest", "image_id", "stop_results",
                       total_warn=be_above_or_equal(0), total_stop=be_above_or_equal(0),
                       last_evaluation=be_an(datetime),
-                      status="pass", image_tag="docker.io/alpine:latest",
+                      status="pass", image_tag=self.image_name,
                       policy_id="default", policy_name="DefaultPolicy",
                       warn_results=not_(be_empty))
         )
 
     with context("but the image has not been scanned yet"):
         with it("returns an error saying that the image has not been found"):
-            image_name = "unknown_image"
-            ok, res = self.client.get_image_scanning_results(image_name)
+            ok, res = self.client.get_image_scanning_results("unknown_image")
 
             expect((ok, res)).to_not(be_successful_api_call)
             expect(res).to(equal("could not retrieve image digest for the given image name, "
                                  "ensure that the image has been scanned"))
 
     with context("but the provided policy id does not exist"):
         with it("returns an error saying that the policy id is not found"):
-            image_name = "alpine"
             policy_id = "unknown_policy_id"
-            ok, res = self.client.get_image_scanning_results(image_name, policy_id)
+            ok, res = self.client.get_image_scanning_results(self.image_name, policy_id)
 
             expect((ok, res)).to_not(be_successful_api_call)
             expect(res).to(equal("the specified policy ID doesn't exist"))

specs/secure/scanning/query_image_content_spec.py

@@ -10,40 +10,41 @@
     with before.each:
         self.client = SdScanningClient(sdc_url=os.getenv("SDC_SECURE_URL", "https://secure.sysdig.com"),
                                        token=os.getenv("SDC_SECURE_TOKEN"))
+        self.image_to_scan = "quay.io/sysdig/agent:latest"
 
     with it("is able to retrieve the OS contents"):
-        ok, res = self.client.query_image_content("alpine:latest", "os")
+        ok, res = self.client.query_image_content(self.image_to_scan, "os")
 
         expect((ok, res)).to(be_successful_api_call)
         expect(res["content"]).to(contain(have_keys("license", "origin", "package", "size", "type", "version")))
         expect(res["content_type"]).to(equal("os"))
 
     with it("is able to retrieve the npm contents"):
-        ok, res = self.client.query_image_content("alpine:latest", "npm")
+        ok, res = self.client.query_image_content(self.image_to_scan, "npm")
 
         expect((ok, res)).to(be_successful_api_call)
         expect(res["content_type"]).to(equal("npm"))
 
     with it("is able to retrieve the gem contents"):
-        ok, res = self.client.query_image_content("alpine:latest", "gem")
+        ok, res = self.client.query_image_content(self.image_to_scan, "gem")
 
         expect((ok, res)).to(be_successful_api_call)
         expect(res["content_type"]).to(equal("gem"))
 
     with it("is able to retrieve the python contents"):
-        ok, res = self.client.query_image_content("alpine:latest", "python")
+        ok, res = self.client.query_image_content(self.image_to_scan, "python")
 
         expect((ok, res)).to(be_successful_api_call)
         expect(res["content_type"]).to(equal("python"))
 
     with it("is able to retrieve the java contents"):
-        ok, res = self.client.query_image_content("alpine:latest", "java")
+        ok, res = self.client.query_image_content(self.image_to_scan, "java")
 
         expect((ok, res)).to(be_successful_api_call)
         expect(res["content_type"]).to(equal("java"))
 
     with it("is able to retrieve the files contents"):
-        ok, res = self.client.query_image_content("alpine:latest", "files")
+        ok, res = self.client.query_image_content(self.image_to_scan, "files")
 
         expect((ok, res)).to(be_successful_api_call)
         expect(res["content"]).to(
@@ -52,7 +53,7 @@
 
     with context("when the type is not in the supported list"):
         with it("returns an error indicating the type is incorrect"):
-            ok, res = self.client.query_image_content("alpine:latest", "Unknown")
+            ok, res = self.client.query_image_content(self.image_to_scan, "Unknown")
 
             expect((ok, res)).not_to(be_successful_api_call)
             expect(res).to(equal(