diff --git a/CHANGELOG.md b/CHANGELOG.md index 0672dcf..c795de2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +- Add UNABLE_TO_DESERIALIZE_RESPONSE http-completion-state. If you have used +`gid.connector.http.source.lookup.continue-on-error` in a previous release, please review your SQL or applications to +account for this new http-completion-state value. + ## [0.23.0] - 2025-11-07 - Ability to specify http versions for http lookups. diff --git a/README.md b/README.md index 53f74a5..a6f4d11 100644 --- a/README.md +++ b/README.md @@ -190,20 +190,24 @@ Metadata columns can be specified and hold http information. They are optional r | Key | Data Type | Description | |-----------------------|----------------------------------|----------------------------------------| -| error-string | STRING NULL | A message associated with the error | +| error-string | STRING NULL | A string associated with the error | | http-status-code | INT NULL | The HTTP status code | | http-headers-map | MAP > NULL | The headers returned with the response | | http-completion-state | STRING NULL | The completion state of the http call. | ##### http-completion-state possible values -| Value | Description | -|:------------------|------------------------| -| SUCCESS | Success | -| HTTP_ERROR_STATUS | HTTP error status code | -| EXCEPTION | An Exception occurred | +| Value | Description | +|:-------------------------------|-------------------------------------| +| SUCCESS | Success | +| HTTP_ERROR_STATUS | HTTP error status code | +| EXCEPTION | An Exception occurred | +| UNABLE_TO_DESERIALIZE_RESPONSE | Unable to deserialize HTTP response | If the `error-string` metadata column is defined on the table and the call succeeds then it will have a null value. +When the HTTP response cannot be deserialized, then the `http-completion-state` will be `UNABLE_TO_DESERIALIZE_RESPONSE` +and the `error-string` will be the response body. Note that `UNABLE_TO_DESERIALIZE_RESPONSE` is a new enum value added +since [0.22.0], please ensure you amend your applications and SQL appropriately. When a http lookup call fails and populates the metadata columns with the error information, the expected enrichment columns from the http call are not populated, this means that they will be null for nullable columns and hold a default value for the type for non-nullable columns. diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpCompletionState.java b/src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpCompletionState.java index 20c1906..2a59992 100644 --- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpCompletionState.java +++ b/src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpCompletionState.java @@ -3,5 +3,6 @@ public enum HttpCompletionState { HTTP_ERROR_STATUS, EXCEPTION, - SUCCESS + SUCCESS, + UNABLE_TO_DESERIALIZE_RESPONSE } diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java b/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java index 81a7567..4338aae 100644 --- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java +++ b/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java @@ -235,8 +235,8 @@ private HttpRowDataWrapper processHttpResponse( rowData = deserialize(responseBody); } catch (IOException e) { if (!this.continueOnError) throw e; - httpCompletionState = HttpCompletionState.EXCEPTION; - errMessage = e.getMessage(); + httpCompletionState = HttpCompletionState.UNABLE_TO_DESERIALIZE_RESPONSE; + errMessage = responseBody; } return HttpRowDataWrapper.builder() .data(rowData) diff --git a/src/test/java/com/getindata/connectors/http/internal/table/lookup/HttpLookupTableSourceITCaseTest.java b/src/test/java/com/getindata/connectors/http/internal/table/lookup/HttpLookupTableSourceITCaseTest.java index 0a093ed..9725b9b 100644 --- a/src/test/java/com/getindata/connectors/http/internal/table/lookup/HttpLookupTableSourceITCaseTest.java +++ b/src/test/java/com/getindata/connectors/http/internal/table/lookup/HttpLookupTableSourceITCaseTest.java @@ -78,6 +78,7 @@ class HttpLookupTableSourceITCaseTest { return row1Id.compareTo(row2Id); }; + public static final String A_TEST_STRING_THAT_IS_NOT_JSON = "A test string that is not json"; private StreamTableEnvironment tEnv; @@ -1081,10 +1082,11 @@ private void assertEnrichedRowsDeserException(Collection collectedRows ) { assertThat(row.getField("balance")).isNull(); // metadata assertThat(row.getField("errStr")) - .isEqualTo("Failed to deserialize JSON 'A test string that is not json'."); + .isEqualTo(A_TEST_STRING_THAT_IS_NOT_JSON); assertThat(row.getField("headers")).isNotNull(); assertThat(row.getField("statusCode")).isEqualTo(200); - assertEquals(row.getField("completionState"), HttpCompletionState.EXCEPTION.name()); + assertEquals(row.getField("completionState"), HttpCompletionState.UNABLE_TO_DESERIALIZE_RESPONSE + .name()); } } ); @@ -1393,7 +1395,7 @@ private void setupServerStubForSpec(TestSpec spec) { if (StringUtils.isNullOrWhitespaceOnly(spec.methodName) || spec.methodName.equalsIgnoreCase("GET")) { wireMockServer.stubFor(get(urlPathEqualTo(ENDPOINT)) .withHeader("Content-Type", equalTo("application/json")) - .willReturn(aResponse().withBody("A test string that is not json").withStatus(200)) + .willReturn(aResponse().withBody(A_TEST_STRING_THAT_IS_NOT_JSON).withStatus(200)) ); } else { setUpServerBodyStub(