Source Trello: get board ids also from organizations (#24266)

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -2071,7 +2071,7 @@
 - name: Trello
   sourceDefinitionId: 8da67652-004c-11ec-9a03-0242ac130003
   dockerRepository: airbyte/source-trello
-  dockerImageTag: 0.3.0
+  dockerImageTag: 0.3.1
   documentationUrl: https://docs.airbyte.com/integrations/sources/trello
   icon: trello.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -15769,7 +15769,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-trello:0.3.0"
+- dockerImage: "airbyte/source-trello:0.3.1"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/trello"
     connectionSpecification:

airbyte-integrations/connectors/source-trello/Dockerfile

@@ -29,5 +29,5 @@ COPY source_trello ./source_trello
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.3.0
+LABEL io.airbyte.version=0.3.1
 LABEL io.airbyte.name=airbyte/source-trello

airbyte-integrations/connectors/source-trello/integration_tests/configured_catalog.json

@@ -68,6 +68,17 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "append",
       "primary_key": [["id"]]
+    },
+    {
+      "stream": {
+        "name": "organizations",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite",
+      "primary_key": [["id"]]
     }
   ]
 }

airbyte-integrations/connectors/source-trello/source_trello/schemas/organizations.json

@@ -0,0 +1,198 @@
+{
+  "$schema": "http://json-schema.org/schema#",
+  "type": "object",
+  "properties": {
+    "id": {
+      "type": ["null", "string"]
+    },
+    "name": {
+      "type": ["null", "string"]
+    },
+    "credits": {
+      "type": "array"
+    },
+    "displayName": {
+      "type": ["null", "string"]
+    },
+    "desc": {
+      "type": ["null", "string"]
+    },
+    "descData": {
+      "type": "object",
+      "properties": {
+        "emoji": {
+          "type": "object"
+        }
+      }
+    },
+    "domainName": {
+      "type": ["null", "string"]
+    },
+    "idBoards": {
+      "type": "array",
+      "items": {
+        "type": ["null", "string"]
+      }
+    },
+    "idMemberCreator": {
+      "type": ["null", "string"]
+    },
+    "invited": {
+      "type": ["null", "boolean"]
+    },
+    "invitations": {
+      "type": "array"
+    },
+    "limits": {
+      "type": "object",
+      "properties": {
+        "orgs": {
+          "type": "object",
+          "properties": {
+            "totalMembersPerOrg": {
+              "type": "object",
+              "properties": {
+                "status": {
+                  "type": ["null", "string"]
+                },
+                "disableAt": {
+                  "type": ["null", "integer"]
+                },
+                "warnAt": {
+                  "type": ["null", "integer"]
+                }
+              }
+            },
+            "freeBoardsPerOrg": {
+              "type": "object",
+              "properties": {
+                "status": {
+                  "type": ["null", "string"]
+                },
+                "disableAt": {
+                  "type": ["null", "integer"]
+                },
+                "warnAt": {
+                  "type": ["null", "integer"]
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "memberships": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "idMember": {
+            "type": ["null", "string"]
+          },
+          "memberType": {
+            "type": ["null", "string"]
+          },
+          "unconfirmed": {
+            "type": ["null", "boolean"]
+          },
+          "deactivated": {
+            "type": ["null", "boolean"]
+          },
+          "id": {
+            "type": ["null", "string"]
+          }
+        }
+      }
+    },
+    "membersCount": {
+      "type": ["null", "integer"]
+    },
+    "prefs": {
+      "type": "object",
+      "properties": {
+        "permissionLevel": {
+          "type": ["null", "string"]
+        },
+        "orgInviteRestrict": {
+          "type": "array"
+        },
+        "boardInviteRestrict": {
+          "type": ["null", "string"]
+        },
+        "externalMembersDisabled": {
+          "type": ["null", "boolean"]
+        },
+        "googleAppsVersion": {
+          "type": ["null", "integer"]
+        },
+        "boardVisibilityRestrict": {
+          "type": "object",
+          "properties": {
+            "private": {
+              "type": ["null", "string"]
+            },
+            "org": {
+              "type": ["null", "string"]
+            },
+            "enterprise": {
+              "type": ["null", "string"]
+            },
+            "public": {
+              "type": ["null", "string"]
+            }
+          }
+        },
+        "boardDeleteRestrict": {
+          "type": "object",
+          "properties": {
+            "private": {
+              "type": ["null", "string"]
+            },
+            "org": {
+              "type": ["null", "string"]
+            },
+            "enterprise": {
+              "type": ["null", "string"]
+            },
+            "public": {
+              "type": ["null", "string"]
+            }
+          }
+        }
+      }
+    },
+    "powerUps": {
+      "type": "array"
+    },
+    "products": {
+      "type": "array"
+    },
+    "billableMemberCount": {
+      "type": ["null", "integer"]
+    },
+    "activeBillableMemberCount": {
+      "type": ["null", "integer"]
+    },
+    "billableCollaboratorCount": {
+      "type": ["null", "integer"]
+    },
+    "url": {
+      "type": ["null", "string"]
+    },
+    "premiumFeatures": {
+      "type": "array",
+      "items": {
+        "type": ["null", "string"]
+      }
+    },
+    "promotions": {
+      "type": "array"
+    },
+    "enterpriseJoinRequest": {
+      "type": "object"
+    },
+    "ixUpdate": {
+      "type": ["null", "string"]
+    }
+  }
+}

airbyte-integrations/connectors/source-trello/source_trello/source.py

@@ -14,7 +14,7 @@
 
 from .auth import TrelloAuthenticator
 from .utils import TrelloRequestRateLimits as balancer
-from .utils import read_full_refresh
+from .utils import read_all_boards
 
 
 class TrelloStream(HttpStream, ABC):
@@ -54,11 +54,11 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
 
 
 class ChildStreamMixin:
-    parent_stream_class: Optional[TrelloStream] = None
-
     def stream_slices(self, sync_mode, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
-        for item in self.parent_stream_class(config=self.config).read_records(sync_mode=sync_mode):
-            yield {"id": item["id"]}
+        board_ids = set(self.config.get("board_ids", []))
+        for board_id in read_all_boards(Boards(self.config), Organizations(self.config)):
+            if not board_ids or board_id in board_ids:
+                yield {"id": board_id}
 
 
 class IncrementalTrelloStream(TrelloStream, ABC):
@@ -112,7 +112,6 @@ class Cards(ChildStreamMixin, TrelloStream):
     Endpoint: https://api.trello.com/1/boards/<id>/cards/all
     """
 
-    parent_stream_class = Boards
     limit = 500
     extra_params = {
         "customFieldItems": "true",
@@ -138,7 +137,6 @@ class Checklists(ChildStreamMixin, TrelloStream):
     Endpoint: https://api.trello.com/1/boards/<id>/checklists
     """
 
-    parent_stream_class = Boards
     extra_params = {"fields": "all", "checkItem_fields": "all"}
 
     def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
@@ -151,20 +149,26 @@ class Lists(ChildStreamMixin, TrelloStream):
     Endpoint: https://api.trello.com/1/boards/<id>/lists
     """
 
-    parent_stream_class = Boards
-
     def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
         return f"boards/{stream_slice['id']}/lists"
 
 
+class Organizations(TrelloStream):
+    """Return list of all member's organizations
+    API Docs: https://developer.atlassian.com/cloud/trello/rest/api-group-members/#api-members-id-organizations-get
+    Endpoint: https://api.trello.com/1/members/me/organizations
+    """
+
+    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
+        return "members/me/organizations"
+
+
 class Users(ChildStreamMixin, TrelloStream):
     """Return list of all members of a boards.
     API Docs: https://developer.atlassian.com/cloud/trello/rest/api-group-boards/#api-boards-id-members-get
     Endpoint: https://api.trello.com/1/boards/<id>/members
     """
 
-    parent_stream_class = Boards
-
     def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
         return f"boards/{stream_slice['id']}/members"
 
@@ -175,7 +179,6 @@ class Actions(ChildStreamMixin, IncrementalTrelloStream):
     Endpoint: https://api.trello.com/1/boards/<id>/actions
     """
 
-    parent_stream_class = Boards
     limit = 1000
 
     def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
@@ -202,8 +205,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
 
         config = self._validate_and_transform(config)
         config["authenticator"] = self._get_authenticator(config)
-        stream = Boards({**config, "board_ids": []})
-        available_boards = {board["id"] for board in read_full_refresh(stream)}
+        available_boards = set(read_all_boards(Boards({**config, "board_ids": []}), Organizations(config)))
         unknown_boards = set(config.get("board_ids", [])) - available_boards
         if unknown_boards:
             unknown_boards = ", ".join(sorted(unknown_boards))
@@ -212,4 +214,4 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         config["authenticator"] = self._get_authenticator(config)
-        return [Actions(config), Boards(config), Cards(config), Checklists(config), Lists(config), Users(config)]
+        return [Actions(config), Boards(config), Cards(config), Checklists(config), Lists(config), Users(config), Organizations(config)]

airbyte-integrations/connectors/source-trello/source_trello/utils.py

@@ -85,3 +85,18 @@ def read_full_refresh(stream_instance: Stream):
         records = stream_instance.read_records(stream_slice=_slice, sync_mode=SyncMode.full_refresh)
         for record in records:
             yield record
+
+
+def read_all_boards(stream_boards: Stream, stream_organizations: Stream):
+    board_ids = set()
+
+    for record in read_full_refresh(stream_boards):
+        if record["id"] not in board_ids:
+            board_ids.add(record["id"])
+            yield record["id"]
+
+    for record in read_full_refresh(stream_organizations):
+        for board_id in record["idBoards"]:
+            if board_id not in board_ids:
+                board_ids.add(board_id)
+                yield board_id

airbyte-integrations/connectors/source-trello/unit_tests/test_source.py

@@ -13,7 +13,7 @@ def test_streams(mocker):
     source = SourceTrello()
     config_mock = MagicMock()
     streams = source.streams(config_mock)
-    expected_streams_number = 6
+    expected_streams_number = 7
     assert len(streams) == expected_streams_number
 
 
@@ -35,6 +35,12 @@ def test_check_connection(requests_mock):
         ],
     )
 
+    requests_mock.get(
+        "https://api.trello.com/1/members/me/organizations",
+        headers=NO_SLEEP_HEADERS,
+        json=[{"id": "org111111111111111111111", "idBoards": ["b11111111111111111111111", "b22222222222222222222222"]}],
+    )
+
     source = SourceTrello()
     status, error = source.check_connection(logger, config)
     assert status is True

airbyte-integrations/connectors/source-trello/unit_tests/test_streams.py

@@ -66,6 +66,12 @@ def test_cards_stream(requests_mock):
         json=[{"id": "b11111111111111111111111", "name": "board_1"}, {"id": "b22222222222222222222222", "name": "board_2"}],
     )
 
+    mock_organizations_request = requests_mock.get(
+        "https://api.trello.com/1/members/me/organizations",
+        headers=NO_SLEEP_HEADERS,
+        json=[{"id": "org111111111111111111111", "idBoards": ["b11111111111111111111111", "b22222222222222222222222"]}],
+    )
+
     json_responses1 = cycle([
         [{"id": "c11111111111111111111111", "name": "card_1"}, {"id": "c22222222222222222222222", "name": "card_2"}],
         [],
@@ -107,5 +113,6 @@ def test_cards_stream(requests_mock):
     assert records == []
 
     assert mock_boards_request.call_count == 3
+    assert mock_organizations_request.call_count == 3
     assert mock_cards_request_1.call_count == 2
     assert mock_cards_request_2.call_count == 4