Fix datapoints insert not reporting failed correctly on `CogniteNotFo…

…undError` (#1728)

CHANGELOG.md

@@ -17,6 +17,11 @@ Changes are grouped as follows
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [7.37.2] - 2024-04-18
+### Fixed
+- Datapoints inserted into non-existent time series, no longer get their identifier hidden in the `failed` attribute
+  on the raised `CogniteNotFoundError`. Any `successful` now also gets reported correctly.
+
 ## [7.37.1] - 2024-04-17
 ### Fixed
 - Updating data set ID now works as expected for `ThreeDModelUpdate`.

cognite/client/_api/datapoints.py

@@ -9,6 +9,7 @@
 from collections import defaultdict
 from datetime import datetime
 from itertools import chain
+from operator import itemgetter
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -1645,6 +1646,7 @@ def insert(self, dps_object_lst: list[dict[str, Any]]) -> None:
         ]
         summary = execute_tasks(self._insert_datapoints, tasks, max_workers=self.max_workers)
         summary.raise_compound_exception_if_failed_tasks(
+            task_unwrap_fn=itemgetter(0),
             task_list_element_unwrap_fn=IdentifierSequenceCore.extract_identifiers,
         )
 
@@ -1740,7 +1742,8 @@ def _insert_datapoints(self, payload: list[dict[str, Any]]) -> None:
             json={"items": payload},
             api_subversion=self.api_subversion,  # TODO: remove once status codes is GA
         )
-        payload.clear()
+        for dct in payload:
+            dct["datapoints"].clear()
 
     @staticmethod
     def _split_datapoints(lst: list[_T], n_first: int, n: int) -> Iterator[tuple[list[_T], bool]]:

cognite/client/_version.py

@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "7.37.1"
+__version__ = "7.37.2"
 __api_subversion__ = "20230101"

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "cognite-sdk"
 
-version = "7.37.1"
+version = "7.37.2"
 description = "Cognite Python SDK"
 readme = "README.md"
 documentation = "https://cognite-sdk-python.readthedocs-hosted.com"

tests/tests_integration/test_api/test_datapoints.py

@@ -2298,6 +2298,23 @@ def test_insert_copy_fails_at_aggregate(self, cognite_client, endpoint_attr, ms_
         with pytest.raises(ValueError, match="Only raw datapoints are supported when inserting data from"):
             cognite_client.time_series.data.insert(data, id=new_ts.id)
 
+    def test_insert_not_found_ts(self, cognite_client, new_ts, monkeypatch):
+        # From 7.35.0 to 7.37.1, 'CogniteNotFoundError.[failed, successful]' was not reported correctly:
+        xid = random_cognite_external_ids(1)[0]
+        dps = [
+            {"id": new_ts.id, "datapoints": [(123456789, 1111111)]},
+            {"external_id": xid, "datapoints": [(123456789, 6666666)]},
+        ]
+        # Let's make sure these two go in separate requests:
+        monkeypatch.setattr(cognite_client.time_series.data, "_POST_DPS_OBJECTS_LIMIT", 1)
+        with pytest.raises(CogniteNotFoundError, match=r"^Not found: \[{") as err:
+            cognite_client.time_series.data.insert_multiple(dps)
+
+        assert isinstance(err.value, CogniteNotFoundError)
+        assert err.value.successful == [{"id": new_ts.id}]
+        assert err.value.not_found == [{"externalId": xid}]
+        assert err.value.failed == [{"externalId": xid}]
+
     @pytest.mark.usefixtures("post_spy")
     def test_insert_pandas_dataframe(self, cognite_client, new_ts, post_spy, monkeypatch):
         df = pd.DataFrame(

tests/tests_integration/test_cognite_client.py

@@ -33,9 +33,25 @@ def test_delete(self, cognite_client):
         assert e.value.code == 404
 
 
+def find_mappingproxy(obj):
+    # Of course this test only fails in CI, so after investigation of where the test pollution
+    # comes from, we may delete this entirely:
+    for k, v in vars(obj).items():
+        try:
+            pickle.dumps(v)
+        except TypeError as e:
+            print(f"failed dumping attribute {k}: {e}")  # noqa T201
+            # We have to go deeper:
+            find_mappingproxy(v)
+
+
 def test_cognite_client_is_picklable(cognite_client):
     if isinstance(cognite_client.config.credentials, (Token, OAuthClientCertificate)):
         pytest.skip()
     # TODO: Test pollution makes this flaky
-    roundtrip_client = pickle.loads(pickle.dumps(cognite_client))
+    try:
+        roundtrip_client = pickle.loads(pickle.dumps(cognite_client))
+    except TypeError:
+        find_mappingproxy(cognite_client)
+        raise
     assert roundtrip_client.iam.token.inspect().projects