🐛 Source Dynamodb: fix list more than 100 tables (#31935)

Co-authored-by: Marcos Marx <marcosmarxm@users.noreply.github.com> Co-authored-by: marcosmarxm <marcosmarxm@gmail.com>
airbytehq · May 1, 2024 · ccfb775 · ccfb775
1 parent 6f69a00
commit ccfb775
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 4 deletions.
diff --git a/airbyte-integrations/connectors/source-dynamodb/metadata.yaml b/airbyte-integrations/connectors/source-dynamodb/metadata.yaml
@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 50401137-8871-4c5a-abb7-1f5fda35545a
-  dockerImageTag: 0.3.0
+  dockerImageTag: 0.3.1
   dockerRepository: airbyte/source-dynamodb
   documentationUrl: https://docs.airbyte.com/integrations/sources/dynamodb
   githubIssueLabel: source-dynamodb

diff --git a/...ce-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbOperations.java b/...ce-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbOperations.java
@@ -22,6 +22,8 @@
 import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
 import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
 import software.amazon.awssdk.services.dynamodb.model.DescribeTableRequest;
+import software.amazon.awssdk.services.dynamodb.model.ListTablesRequest;
+import software.amazon.awssdk.services.dynamodb.model.ListTablesResponse;
 import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
 
 public class DynamodbOperations extends AbstractDatabase implements Closeable {
@@ -56,9 +58,24 @@ private void initMappers() {
   }
 
   public List<String> listTables() {
-    return dynamoDbClient.listTables()
-        // filter on table status?
-        .tableNames();
+    List<String> tableNames = new ArrayList<>();
+    ListTablesRequest listTablesRequest = ListTablesRequest.builder().build();
+    boolean completed = false;
+
+    while (!completed) {
+      ListTablesResponse listTablesResponse = dynamoDbClient.listTables(listTablesRequest);
+      tableNames.addAll(listTablesResponse.tableNames());
+
+      if (listTablesResponse.lastEvaluatedTableName() == null) {
+        completed = true;
+      } else {
+        listTablesRequest = listTablesRequest.toBuilder()
+            .exclusiveStartTableName(listTablesResponse.lastEvaluatedTableName())
+            .build();
+      }
+    }
+
+    return tableNames;
   }
 
   public List<String> primaryKey(String tableName) {

diff --git a/docs/integrations/sources/dynamodb.md b/docs/integrations/sources/dynamodb.md
@@ -75,6 +75,7 @@ the underlying role executing the container workload in AWS.
 
 | Version | Date       | Pull Request                                              | Subject                                                                |
 |:--------| :--------- | :-------------------------------------------------------- |:-----------------------------------------------------------------------|
+| 0.3.1   | 2024-05-01 | [31935](https://github.com/airbytehq/airbyte/pull/31935) | Fix list more than 100 tables |
 | 0.3.0   | 2024-04-24 | [37530](https://github.com/airbytehq/airbyte/pull/37530) | Allow role based access                                                |
 | 0.2.3   | 2024-02-13 | [35232](https://github.com/airbytehq/airbyte/pull/35232) | Adopt CDK 0.20.4                                                       |
 | 0.2.2   | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version                                                       |