fix: Send entire JSON, to send the Content-Length header [DAT-818]

tests/test_datasource.py

@@ -44,6 +44,28 @@ def handler(request):
         assert response.ok
 
 
+    def test_append_ndjson(self, httpserver: HTTPServer):
+        ds = Datasource("mydatasource", "123456", api=httpserver.url_for("/"))
+
+        records = [{"key": "foo", "value": "bar"}, {"key": "baz", "value": "ed"}]
+        expected_body = '{"key": "foo", "value": "bar"}\n{"key": "baz", "value": "ed"}\n'
+
+        httpserver.expect_request(
+            "/v0/datasources",
+            query_string={
+                "name": "mydatasource",
+                "mode": "append",
+                "format": "ndjson",
+            },
+            data=expected_body,
+            headers={"Content-Length": str(len(expected_body.encode()))},
+        ).respond_with_data("", 200)
+
+        response = ds.append_ndjson(records)
+        httpserver.check()
+        assert response.ok
+
+
 class TestFileDatasource:
     def test_append(self, tmp_path):
         file_path = tmp_path / "myfile.csv"

verdin/datasource.py

@@ -147,9 +147,7 @@ def append_ndjson(self, records: List[Dict]) -> requests.Response:
         :return: The HTTP response
         """
 
-        def _ndjson_iterator():
-            for record in records:
-                yield json.dumps(record) + "\n"
+        data = "".join(json.dumps(record) + "\n" for record in records)
 
         LOG.debug(
             "appending %d ndjson records to %s via %s",
@@ -160,7 +158,7 @@ def _ndjson_iterator():
         response = self._datasources_api.append(
             name=self.canonical_name,
             format="ndjson",
-            data=_ndjson_iterator(),
+            data=data,
         )
 
         return response._response