Improve error handling when additional sources/destinations cannot be…

… read (airbytehq#8031) * Improve error handling when additional sources/destinations cannot be read. * Whitespace
schlattk · Jan 4, 2022 · e396711 · e396711
1 parent a827eef
commit e396711
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 2 deletions.
diff --git a/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java b/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java
@@ -53,7 +53,7 @@ public AirbyteGithubStore(final String baseUrl, final Duration timeout) {
   public List<StandardDestinationDefinition> getLatestDestinations() throws InterruptedException {
     try {
       return YamlListToStandardDefinitions.toStandardDestinationDefinitions(getFile(DESTINATION_DEFINITION_LIST_LOCATION_PATH));
-    } catch (final IOException e) {
+    } catch (final Throwable e) {
       LOGGER.warn(
           "Unable to retrieve latest Destination list from Github. Using the list bundled with Airbyte. This warning is expected if this Airbyte cluster does not have internet access.",
           e);
@@ -64,7 +64,7 @@ public List<StandardDestinationDefinition> getLatestDestinations() throws Interr
   public List<StandardSourceDefinition> getLatestSources() throws InterruptedException {
     try {
       return YamlListToStandardDefinitions.toStandardSourceDefinitions(getFile(SOURCE_DEFINITION_LIST_LOCATION_PATH));
-    } catch (final IOException e) {
+    } catch (final Throwable e) {
       LOGGER.warn(
           "Unable to retrieve latest Source list from Github. Using the list bundled with Airbyte. This warning is expected if this Airbyte cluster does not have internet access.",
           e);

diff --git a/airbyte-server/src/test/java/io/airbyte/server/services/AirbyteGithubStoreTest.java b/airbyte-server/src/test/java/io/airbyte/server/services/AirbyteGithubStoreTest.java
@@ -32,6 +32,56 @@ public void setUp() {
     githubStore = AirbyteGithubStore.test(webServer.url("/").toString(), TIMEOUT);
   }
 
+  @Nested
+  @DisplayName("when the additional definitions file is unusable, badly formatted, or cannot be retrieved due to errors")
+  class FileUnusable {
+
+    @Test
+    void testGetLatestSourcesWithNonJson() throws InterruptedException {
+      final var nonjsonBody = "irrelevant text";
+      final var nonjsonResponse = new MockResponse().setResponseCode(200)
+          .addHeader("Content-Type", "text/plain; charset=utf-8")
+          .addHeader("Cache-Control", "no-cache")
+          .setBody(nonjsonBody);
+      webServer.enqueue(nonjsonResponse);
+      assertEquals(Collections.emptyList(), githubStore.getLatestSources());
+    }
+
+    @Test
+    void testGetLatestSourcesWithWrongSchemaJson() throws InterruptedException {
+      final var jsonBody = "{ json: 'validButWrongFormat' }";
+      final var jsonResponse = new MockResponse().setResponseCode(200)
+          .addHeader("Content-Type", "application/json; charset=utf-8")
+          .addHeader("Cache-Control", "no-cache")
+          .setBody(jsonBody);
+      webServer.enqueue(jsonResponse);
+      assertEquals(Collections.emptyList(), githubStore.getLatestSources());
+    }
+
+    @Test
+    void testGetLatestDestinationsWithNonJson() throws InterruptedException {
+      final var nonjsonBody = "irrelevant text";
+      final var nonjsonResponse = new MockResponse().setResponseCode(200)
+          .addHeader("Content-Type", "text/plain; charset=utf-8")
+          .addHeader("Cache-Control", "no-cache")
+          .setBody(nonjsonBody);
+      webServer.enqueue(nonjsonResponse);
+      assertEquals(Collections.emptyList(), githubStore.getLatestDestinations());
+    }
+
+    @Test
+    void testGetLatestDestinationsWithWrongSchemaJson() throws InterruptedException {
+      final var jsonBody = "{ json: 'validButWrongFormat' }";
+      final var jsonResponse = new MockResponse().setResponseCode(200)
+          .addHeader("Content-Type", "application/json; charset=utf-8")
+          .addHeader("Cache-Control", "no-cache")
+          .setBody(jsonBody);
+      webServer.enqueue(jsonResponse);
+      assertEquals(Collections.emptyList(), githubStore.getLatestDestinations());
+    }
+
+  }
+
   @Nested
   @DisplayName("when there is no internet")
   class NoInternet {