Source Salesforce: add unit test for encoding (#29622)

airbytehq · Aug 18, 2023 · 4ac3322 · 4ac3322
1 parent 752b42a
commit 4ac3322
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 5 deletions.
diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py
@@ -282,16 +282,16 @@ def test_read_with_chunks_should_return_null_value_when_no_data_is_provided(stre
 
 
 @pytest.mark.parametrize(
-    "chunk_size, content_type, content, expected_result",
+    "chunk_size, content_type_header, content, expected_result",
     encoding_symbols_parameters(),
     ids=[f"charset: {x[1]}, chunk_size: {x[0]}" for x in encoding_symbols_parameters()],
 )
-def test_encoding_symbols(stream_config, stream_api, chunk_size, content_type, content, expected_result):
+def test_encoding_symbols(stream_config, stream_api, chunk_size, content_type_header, content, expected_result):
     job_full_url_results: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA/results"
     stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api)
 
     with requests_mock.Mocker() as m:
-        m.register_uri("GET", job_full_url_results, headers={"Content-Type": f"text/html; charset={content_type}"}, content=content)
+        m.register_uri("GET", job_full_url_results, headers=content_type_header, content=content)
         tmp_file, response_encoding, _ = stream.download_data(url=job_full_url_results)
         res = list(stream.read_with_chunks(tmp_file, response_encoding))
         assert res == expected_result

diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py
@@ -130,10 +130,26 @@ def generate_stream(stream_name, stream_config, stream_api):
 
 
 def encoding_symbols_parameters():
-    return [(x, "ISO-8859-1", b'"\xc4"\n,"4"\n\x00,"\xca \xfc"', [{"Ä": "4"}, {"Ä": "Ê ü"}]) for x in range(1, 11)] + [
+    return [(x, {"Content-Type": "text/csv; charset=ISO-8859-1"}, b'"\xc4"\n,"4"\n\x00,"\xca \xfc"', [{"Ä": "4"}, {"Ä": "Ê ü"}]) for x in range(1, 11)] + [
         (
             x,
-            "utf-8",
+            {"Content-Type": "text/csv; charset=utf-8"},
+            b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"',
+            [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}],
+        )
+        for x in range(1, 11)
+    ] + [
+        (
+            x,
+            {"Content-Type": "text/csv"},
+            b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"',
+            [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}],
+        )
+        for x in range(1, 11)
+    ] + [
+        (
+            x,
+            {},
             b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"',
             [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}],
         )