Feature/integ 1737 xfc support (#352)

CHANGELOG.md

@@ -14,6 +14,8 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta
 
 - `RiskIndicator` and `RiskSeverity` filter classes to new `py42.sdk.queries.fileevents.filters.risk_filter` module.
 
+- Support for Incydr SKUs that do not include backup and restore functionality to `sdk.securitydata.stream_file_by_sha256()` and  `sdk.securitydata.stream_file_by_md5()`.
+
 - New method `sdk.alerts.get_all_alert_details()` as a helper to make getting alerts with details easier (combines `sdk.alerts.search_all_pages()` and `sdk.alerts.get_details()`).
 
 ### Removed

src/py42/clients/securitydata.py

@@ -251,17 +251,9 @@ def _stream_file(self, checksum, version_info):
             device_guid, md5_hash, sha256_hash, path
         )
         if version:
-            pds = self._storage_service_factory.create_preservation_data_service(
-                version["storageNodeURL"]
-            )
-            token = pds.get_download_token(
-                version["archiveGuid"], version["fileId"], version["versionTimestamp"],
-            )
-            return pds.get_file(str(token))
+            return self._get_file_stream(version)
 
-        raise Py42Error(
-            f"No file with hash {checksum} available for download on any storage node."
-        )
+        raise Py42Error(f"No file with hash {checksum} available for download.")
 
     def _get_file_version_for_stream(self, device_guid, md5_hash, sha256_hash, path):
         version = self._get_device_file_version(
@@ -275,18 +267,17 @@ def _get_device_file_version(self, device_guid, md5_hash, sha256_hash, path):
         response = self._preservation_data_service.get_file_version_list(
             device_guid, md5_hash, sha256_hash, path
         )
-        versions = response["versions"]
+        versions = (
+            response.data.get("securityEventVersionsMatchingChecksum")
+            or response.data.get("securityEventVersionsAtPath")
+            or response.data.get("preservationVersions")
+        )
+
         if versions:
-            exact_match = next(
-                (
-                    x
-                    for x in versions
-                    if x["fileMD5"] == md5_hash and x["fileSHA256"] == sha256_hash
-                ),
-                None,
-            )
-            if exact_match:
-                return exact_match
+            if not response.data.get("securityEventVersionsAtPath"):
+                exact_match = _get_first_matching_version(versions, md5_hash)
+                if exact_match:
+                    return exact_match
 
             most_recent = sorted(
                 versions, key=lambda i: i["versionTimestamp"], reverse=True
@@ -303,8 +294,35 @@ def _get_other_file_location_version(self, md5_hash, sha256_hash):
             version = self._preservation_data_service.find_file_version(
                 md5_hash, sha256_hash, paths
             )
-            if version.status_code != 204:
-                return version
+            if version.status_code == 200:
+                return version.data
+
+    def _get_file_stream(self, version):
+        if version.get("edsUrl"):
+            return self._get_exfiltrated_file(version)
+
+        return self._get_stored_file(version)
+
+    def _get_exfiltrated_file(self, version):
+        eds = self._storage_service_factory.create_exfiltrated_data_service(
+            version["edsUrl"]
+        )
+        token = eds.get_download_token(
+            version["eventId"],
+            version["deviceUid"],
+            version["filePath"],
+            version["versionTimestamp"],
+        )
+        return eds.get_file(str(token))
+
+    def _get_stored_file(self, version):
+        pds = self._storage_service_factory.create_preservation_data_service(
+            version["storageNodeURL"]
+        )
+        token = pds.get_download_token(
+            version["archiveGuid"], version["fileId"], version["versionTimestamp"],
+        )
+        return pds.get_file(str(token))
 
     def _get_plan_storage_infos(self, plan_destination_map):
         plan_infos = []
@@ -466,6 +484,12 @@ def node_guid(self):
         return self._node_guid
 
 
+def _get_first_matching_version(versions, md5_hash):
+    exact_match = next((x for x in versions if x["fileMD5"] == md5_hash), None)
+    if exact_match:
+        return exact_match
+
+
 def _escape_quote_chars_in_token(token):
     """
     The `nextPgToken` returned in Forensic Search requests with > 10k results is the eventId

src/py42/services/preservationdata.py

@@ -20,7 +20,8 @@ def find_file_version(self, file_md5, file_sha256, paths):
         uri = "/api/v1/FindAvailableVersion"
         return self._connection.post(uri, json=data)
 
-    def get_file_version_list(self, deviceGuid, file_md5, file_sha256, path):
-        params = f"fileSHA256={file_sha256}&fileMD5={file_md5}&deviceGuid={deviceGuid}&path={quote(path)}"
-        uri = f"/api/v1/FileVersionListing?{params}"
+    def get_file_version_list(self, device_id, file_md5, file_sha256, path):
+        params = "fileSHA256={}&fileMD5={}&deviceUid={}&filePath={}"
+        params = params.format(file_sha256, file_md5, device_id, quote(path))
+        uri = f"/api/v2/file-version-listing?{params}"
         return self._connection.get(uri)

src/py42/services/storage/_service_factory.py

@@ -5,6 +5,7 @@
 from py42.services.storage._auth import FileArchiveAuth
 from py42.services.storage._auth import SecurityArchiveAuth
 from py42.services.storage.archive import StorageArchiveService
+from py42.services.storage.exfiltrateddata import ExfiltratedDataService
 from py42.services.storage.preservationdata import StoragePreservationDataService
 from py42.services.storage.restore import PushRestoreService
 from py42.services.storage.securitydata import StorageSecurityDataService
@@ -35,6 +36,11 @@ def create_preservation_data_service(self, host_address):
         streaming_connection = Connection.from_host_address(host_address)
         return StoragePreservationDataService(main_connection, streaming_connection)
 
+    def create_exfiltrated_data_service(self, host_address):
+        main_connection = self._connection.clone(host_address)
+        streaming_connection = Connection.from_host_address(host_address)
+        return ExfiltratedDataService(main_connection, streaming_connection)
+
     def auto_select_destination_guid(self, device_guid):
         response = self._device_service.get_by_guid(
             device_guid, include_backup_usage=True

src/py42/services/storage/exfiltrateddata.py

@@ -0,0 +1,47 @@
+from urllib.parse import quote
+
+from py42.services import BaseService
+
+
+class ExfiltratedDataService(BaseService):
+
+    _base_uri = "api/v1/"
+
+    def __init__(self, main_session, streaming_session):
+        super().__init__(main_session)
+        self._streaming_session = streaming_session
+
+    def get_download_token(self, event_id, device_id, file_path, timestamp):
+        """Get EDS download token for a file.
+
+        Args:
+            event_id (str): Id of the file event that references the file desired for download.
+            device_id (str): Id of the device on which the file desired for download is stored.
+            file_path (str): Path where the file desired for download resides on the device.
+            timestamp (int): Last updated timestamp of the file in milliseconds.
+
+        Returns:
+            :class:`py42.response.Py42Response`: A response containing download token for the file.
+        """
+        params = "deviceUid={}&eventId={}&filePath={}&versionTimestamp={}"
+        params = params.format(device_id, event_id, quote(file_path), timestamp)
+        resource = "file-download-token"
+        uri = f"{self._base_uri}{resource}?{params}"
+        return self._connection.get(uri)
+
+    def get_file(self, token):
+        """Streams a file.
+
+        Args:
+            token (str):EDS Download token.
+
+        Returns:
+            Returns a stream of the file indicated by the input token.
+        """
+        resource = "get-file"
+        uri = f"{self._connection.host_address}/{self._base_uri}{resource}"
+        params = {"token": token}
+        headers = {"Accept": "*/*"}
+        return self._streaming_session.get(
+            uri, params=params, headers=headers, stream=True
+        )

tests/clients/_archiveaccess/conftest.py

@@ -1,11 +1,9 @@
-import json
-
 import pytest
+from tests.conftest import create_mock_response
 from tests.conftest import TEST_SESSION_ID
 
 from py42.clients._archiveaccess.restoremanager import FileSizePoller
 from py42.clients._archiveaccess.restoremanager import RestoreJobManager
-from py42.response import Py42Response
 from py42.services.storage.archive import StorageArchiveService
 from py42.services.storage.restore import PushRestoreService
 
@@ -18,13 +16,10 @@ def push_service(mocker):
 @pytest.fixture
 def storage_archive_service(mocker):
     client = mocker.MagicMock(spec=StorageArchiveService)
-    py42_response = mocker.MagicMock(spec=Py42Response)
-    py42_response.text = f'{{"webRestoreSessionId": "{TEST_SESSION_ID}"}}'
-    py42_response.status_code = 200
-    py42_response.encoding = None
-    py42_response.__getitem__ = lambda _, key: json.loads(py42_response.text).get(key)
-
-    client.create_restore_session.return_value = py42_response
+    response = create_mock_response(
+        mocker, f'{{"webRestoreSessionId": "{TEST_SESSION_ID}"}}'
+    )
+    client.create_restore_session.return_value = response
     return client
 
 

tests/clients/_archiveaccess/test_restoremanager.py

@@ -1,7 +1,7 @@
 import json
 import time
 
-from requests import Response
+from tests.conftest import create_mock_response
 from tests.conftest import TEST_ACCEPTING_GUID
 from tests.conftest import TEST_BACKUP_SET_ID
 from tests.conftest import TEST_DEVICE_GUID
@@ -74,21 +74,15 @@ def mock_start_restore(
         num_bytes,
         **kwargs
     ):
-        start_restore_response = mocker.MagicMock(spec=Response)
-        start_restore_response.text = response
-        start_restore_response.status_code = 200
-        return Py42Response(start_restore_response)
+        return create_mock_response(mocker, response)
 
     storage_archive_service.start_restore.side_effect = mock_start_restore
 
 
 def mock_get_restore_status_responses(mocker, storage_archive_service, json_responses):
     responses = []
     for json_response in json_responses:
-        get_restore_status_response = mocker.MagicMock(spec=Response)
-        get_restore_status_response.text = json_response
-        get_restore_status_response.status_code = 200
-        responses.append(Py42Response(get_restore_status_response))
+        responses.append(create_mock_response(mocker, json_response))
 
     storage_archive_service.get_restore_status.side_effect = responses
 
@@ -119,28 +113,26 @@ class TestFileSizePoller:
 
     def get_create_job_side_effect(self, mocker):
         def create_job(guid, file_id, *args, **kwargs):
-            resp = mocker.MagicMock(spec=Response)
             if file_id == TEST_DOWNLOADS_FILE_ID:
-                resp.text = json.dumps({"jobId": self.DOWNLOADS_JOB})
+                text = json.dumps({"jobId": self.DOWNLOADS_JOB})
             elif file_id == TEST_DOWNLOADS_DIR_ID:
-                resp.text = json.dumps({"jobId": self.EXT_JOB})
+                text = json.dumps({"jobId": self.EXT_JOB})
 
-            return Py42Response(resp)
+            return create_mock_response(mocker, text)
 
         return create_job
 
     def get_file_sizes_polling_status_side_effect(
         self, mocker,
     ):
         def get_status(job_id, device_guid):
-            resp = mocker.MagicMock(spec=Response)
             if job_id == self.DOWNLOADS_JOB:
                 self.EXTERNAL_DIR_SIZES["status"] = "DONE"
-                resp.text = json.dumps(self.EXTERNAL_DIR_SIZES)
+                text = json.dumps(self.EXTERNAL_DIR_SIZES)
             elif job_id == self.EXT_JOB:
                 self.DOWNLOADS_DIR_SIZES["status"] = "DONE"
-                resp.text = json.dumps(self.DOWNLOADS_DIR_SIZES)
-            return Py42Response(resp)
+                text = json.dumps(self.DOWNLOADS_DIR_SIZES)
+            return create_mock_response(mocker, text)
 
         return get_status
 
@@ -184,16 +176,14 @@ def test_get_file_sizes_waits_for_size_calculation(
         desktop_statuses = ["DONE", "WORKING", "WORKING"]
 
         def get_file_sizes(job_id, device_id):
-            resp = mocker.MagicMock(spec=Response)
             if job_id == self.DOWNLOADS_JOB:
                 status = desktop_statuses.pop()
                 self.EXTERNAL_DIR_SIZES["status"] = status
-                resp.text = json.dumps(self.EXTERNAL_DIR_SIZES)
 
             elif job_id == self.EXT_JOB:
                 self.EXTERNAL_DIR_SIZES["status"] = "DONE"
-                resp.text = json.dumps(self.EXTERNAL_DIR_SIZES)
-            return Py42Response(resp)
+
+            return create_mock_response(mocker, json.dumps(self.EXTERNAL_DIR_SIZES))
 
         storage_archive_service.get_file_size_job.side_effect = get_file_sizes
         poller = FileSizePoller(storage_archive_service, TEST_DEVICE_GUID)

tests/clients/test_alertrules.py

@@ -1,13 +1,12 @@
 import json
 
 import pytest
-from requests import Response
 from tests.conftest import create_mock_error
+from tests.conftest import create_mock_response
 
 from py42.clients.alertrules import AlertRulesClient
 from py42.exceptions import Py42InternalServerError
 from py42.exceptions import Py42InvalidRuleOperationError
-from py42.response import Py42Response
 from py42.services.alertrules import AlertRulesService
 from py42.services.alerts import AlertService
 
@@ -29,10 +28,8 @@
 
 @pytest.fixture
 def mock_alerts_service_system_rule(mocker, mock_alerts_service):
-    response = mocker.MagicMock(spec=Response)
-    response.text = json.dumps(TEST_SYSTEM_RULE_RESPONSE)
-    py42_response = Py42Response(response)
-    mock_alerts_service.get_rule_by_observer_id.return_value = py42_response
+    response = create_mock_response(mocker, json.dumps(TEST_SYSTEM_RULE_RESPONSE))
+    mock_alerts_service.get_rule_by_observer_id.return_value = response
     return mock_alerts_service