diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0577c2653a210..7b3b006301599 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -2071,7 +2071,7 @@ - name: Trello sourceDefinitionId: 8da67652-004c-11ec-9a03-0242ac130003 dockerRepository: airbyte/source-trello - dockerImageTag: 0.3.0 + dockerImageTag: 0.3.1 documentationUrl: https://docs.airbyte.com/integrations/sources/trello icon: trello.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 242ba5ea4ef4f..b0cf2624674c8 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -15769,7 +15769,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-trello:0.3.0" +- dockerImage: "airbyte/source-trello:0.3.1" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/trello" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-trello/Dockerfile b/airbyte-integrations/connectors/source-trello/Dockerfile index 1024ec8161d85..a1d37886b8056 100644 --- a/airbyte-integrations/connectors/source-trello/Dockerfile +++ b/airbyte-integrations/connectors/source-trello/Dockerfile @@ -29,5 +29,5 @@ COPY source_trello ./source_trello ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.3.1 LABEL io.airbyte.name=airbyte/source-trello diff --git a/airbyte-integrations/connectors/source-trello/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-trello/integration_tests/configured_catalog.json index d8b3d1866b176..4486640726891 100644 --- a/airbyte-integrations/connectors/source-trello/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-trello/integration_tests/configured_catalog.json @@ -68,6 +68,17 @@ "sync_mode": "full_refresh", "destination_sync_mode": "append", "primary_key": [["id"]] + }, + { + "stream": { + "name": "organizations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] } ] } diff --git a/airbyte-integrations/connectors/source-trello/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-trello/integration_tests/expected_records.jsonl index 6322d320bc508..273ddb82dc7a9 100644 --- a/airbyte-integrations/connectors/source-trello/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-trello/integration_tests/expected_records.jsonl @@ -1,6 +1,3 @@ -{"stream":"actions","data":{"id":"6413643417610bdd29d36c47","idMemberCreator":"610be3762899a26d04256dae","data":{"card":{"name":"test_1 (fixed)","id":"611aa1eae05ae943ada67f8a","idShort":3,"shortLink":"FhvCm05w"},"old":{"name":"test_1"},"board":{"id":"611aa0ef37acd675af67dc9b","name":"test_board","shortLink":"Lj11WRiJ"},"list":{"id":"611aa0ef37acd675af67dc9c","name":"To Do"}},"appCreator":null,"type":"updateCard","date":"2023-03-16T18:47:16.404Z","limits":null,"memberCreator":{"id":"610be3762899a26d04256dae","activityBlocked":false,"avatarHash":"ba09ec53715b75fd24683040c248de6e","avatarUrl":"https://trello-members.s3.amazonaws.com/610be3762899a26d04256dae/ba09ec53715b75fd24683040c248de6e","fullName":"integration test","idMemberReferrer":null,"initials":"IT","nonPublic":{},"nonPublicAvailable":true,"username":"integrationtest19"}},"emitted_at":1679051289446} -{"stream":"actions","data":{"id":"6127a34a824394333c4b089e","idMemberCreator":"610be3762899a26d04256dae","data":{"old":{"closed":false},"card":{"closed":true,"id":"6127a3488a027b342502ba23","name":"Demo_test","idShort":7,"shortLink":"bbhWatIg"},"board":{"id":"611aa0ef37acd675af67dc9b","name":"test_board","shortLink":"Lj11WRiJ"},"list":{"id":"611aa0ef37acd675af67dc9d","name":"Doing"}},"appCreator":{"id":"56ff07afbc287582d36e19ac","name":"Butler","icon":{"url":"https://app.butlerfortrello.com/assets/icon-butler-bot.png"}},"type":"updateCard","date":"2021-08-26T14:20:58.523Z","limits":null,"memberCreator":{"id":"610be3762899a26d04256dae","activityBlocked":false,"avatarHash":"ba09ec53715b75fd24683040c248de6e","avatarUrl":"https://trello-members.s3.amazonaws.com/610be3762899a26d04256dae/ba09ec53715b75fd24683040c248de6e","fullName":"integration test","idMemberReferrer":null,"initials":"IT","nonPublic":{},"nonPublicAvailable":true,"username":"integrationtest19"}},"emitted_at":1679051289446} -{"stream":"actions","data":{"id":"6127a34ac4dc0b1d29f7667f","idMemberCreator":"610be3762899a26d04256dae","data":{"old":{"pos":131071},"card":{"pos":32767.5,"id":"6127a3488a027b342502ba23","name":"Demo_test","idShort":7,"shortLink":"bbhWatIg"},"board":{"id":"611aa0ef37acd675af67dc9b","name":"test_board","shortLink":"Lj11WRiJ"},"list":{"id":"611aa0ef37acd675af67dc9d","name":"Doing"}},"appCreator":{"id":"56ff07afbc287582d36e19ac","name":"Butler","icon":{"url":"https://app.butlerfortrello.com/assets/icon-butler-bot.png"}},"type":"updateCard","date":"2021-08-26T14:20:58.193Z","limits":null,"memberCreator":{"id":"610be3762899a26d04256dae","activityBlocked":false,"avatarHash":"ba09ec53715b75fd24683040c248de6e","avatarUrl":"https://trello-members.s3.amazonaws.com/610be3762899a26d04256dae/ba09ec53715b75fd24683040c248de6e","fullName":"integration test","idMemberReferrer":null,"initials":"IT","nonPublic":{},"nonPublicAvailable":true,"username":"integrationtest19"}},"emitted_at":1679051289447} {"stream":"boards","data":{"id":"611aa0ef37acd675af67dc9b","nodeId":"ari:cloud:trello::board/workspace/610be50a0537086c571a5684/611aa0ef37acd675af67dc9b","name":"test_board","desc":"","descData":null,"closed":false,"dateClosed":null,"idOrganization":"610be50a0537086c571a5684","idEnterprise":null,"limits":{"attachments":{"perBoard":{"status":"ok","disableAt":36000,"warnAt":28800},"perCard":{"status":"ok","disableAt":1000,"warnAt":800}},"boards":{"totalMembersPerBoard":{"status":"ok","disableAt":1600,"warnAt":1280},"totalAccessRequestsPerBoard":{"status":"ok","disableAt":4000,"warnAt":3200}},"cards":{"openPerBoard":{"status":"ok","disableAt":5000,"warnAt":4000},"openPerList":{"status":"ok","disableAt":5000,"warnAt":4000},"totalPerBoard":{"status":"ok","disableAt":2000000,"warnAt":1600000},"totalPerList":{"status":"ok","disableAt":1000000,"warnAt":800000}},"checklists":{"perBoard":{"status":"ok","disableAt":1800000,"warnAt":1440000},"perCard":{"status":"ok","disableAt":500,"warnAt":400}},"checkItems":{"perChecklist":{"status":"ok","disableAt":200,"warnAt":160}},"customFields":{"perBoard":{"status":"ok","disableAt":50,"warnAt":40}},"customFieldOptions":{"perField":{"status":"ok","disableAt":50,"warnAt":40}},"labels":{"perBoard":{"status":"ok","disableAt":1000,"warnAt":800}},"lists":{"openPerBoard":{"status":"ok","disableAt":500,"warnAt":400},"totalPerBoard":{"status":"ok","disableAt":3000,"warnAt":2400}},"stickers":{"perCard":{"status":"ok","disableAt":70,"warnAt":56}},"reactions":{"perAction":{"status":"ok","disableAt":900,"warnAt":720},"uniquePerAction":{"status":"ok","disableAt":17,"warnAt":14}}},"pinned":false,"starred":false,"url":"https://trello.com/b/Lj11WRiJ/testboard","prefs":{"permissionLevel":"org","hideVotes":false,"voting":"disabled","comments":"members","invitations":"members","selfJoin":true,"cardCovers":true,"isTemplate":false,"cardAging":"regular","calendarFeedEnabled":false,"hiddenPluginBoardButtons":[],"switcherViews":[{"_id":"63dee894b0ed3558c379bcf1","viewType":"Board","enabled":true,"typeName":"SwitcherViews","id":"63dee894b0ed3558c379bcf1"},{"_id":"63dee894b0ed3558c379bcf2","viewType":"Table","enabled":true,"typeName":"SwitcherViews","id":"63dee894b0ed3558c379bcf2"},{"_id":"63dee894b0ed3558c379bcf3","viewType":"Calendar","enabled":false,"typeName":"SwitcherViews","id":"63dee894b0ed3558c379bcf3"},{"_id":"63dee894b0ed3558c379bcf4","viewType":"Dashboard","enabled":false,"typeName":"SwitcherViews","id":"63dee894b0ed3558c379bcf4"},{"_id":"63dee894b0ed3558c379bcf5","viewType":"Timeline","enabled":false,"typeName":"SwitcherViews","id":"63dee894b0ed3558c379bcf5"},{"_id":"63dee894b0ed3558c379bcf6","viewType":"Map","enabled":false,"typeName":"SwitcherViews","id":"63dee894b0ed3558c379bcf6"}],"background":"611a97ed741bfb5782acf105","backgroundColor":null,"backgroundImage":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/original/a2614b98135f746b4b1d38a1eef0240c/photo-1629006307992-dc1891987730","backgroundImageScaled":[{"width":67,"height":100,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/67x100/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":128,"height":192,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/128x192/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":320,"height":480,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/320x480/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":640,"height":960,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/640x960/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":683,"height":1024,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/683x1024/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":854,"height":1280,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/854x1280/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":1067,"height":1600,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/1067x1600/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":1281,"height":1920,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/1281x1920/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":1366,"height":2048,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/original/a2614b98135f746b4b1d38a1eef0240c/photo-1629006307992-dc1891987730"}],"backgroundTile":false,"backgroundBrightness":"light","backgroundBottomColor":"#7c8280","backgroundTopColor":"#aeb0ae","canBePublic":true,"canBeEnterprise":true,"canBeOrg":true,"canBePrivate":true,"canInvite":true},"shortLink":"Lj11WRiJ","subscribed":false,"labelNames":{"green":"","yellow":"","orange":"","red":"","purple":"","blue":"","sky":"","lime":"","pink":"","black":"","green_dark":"","yellow_dark":"","orange_dark":"","red_dark":"","purple_dark":"","blue_dark":"","sky_dark":"","lime_dark":"","pink_dark":"","black_dark":"","green_light":"","yellow_light":"","orange_light":"","red_light":"","purple_light":"","blue_light":"","sky_light":"","lime_light":"","pink_light":"","black_light":""},"powerUps":[],"dateLastActivity":"2023-03-16T18:47:16.386Z","dateLastView":"2023-03-16T20:10:44.953Z","shortUrl":"https://trello.com/b/Lj11WRiJ","idTags":[],"datePluginDisable":null,"creationMethod":"automatic","ixUpdate":"90","templateGallery":null,"enterpriseOwned":false,"idBoardSource":null,"premiumFeatures":["additionalBoardBackgrounds","additionalStickers","customBoardBackgrounds","customEmoji","customStickers","plugins"],"idMemberCreator":"610be3762899a26d04256dae","memberships":[{"idMember":"610be3762899a26d04256dae","memberType":"admin","unconfirmed":false,"deactivated":false,"id":"611aa0ef37acd675af67dc9f"}]},"emitted_at":1679051289978} {"stream":"boards","data":{"id":"611aa586ef5f2c8e1deec8b6","nodeId":"ari:cloud:trello::board/workspace/610be50a0537086c571a5684/611aa586ef5f2c8e1deec8b6","name":"test_board_new","desc":"","descData":null,"closed":false,"dateClosed":null,"idOrganization":"610be50a0537086c571a5684","idEnterprise":null,"limits":{"attachments":{"perBoard":{"status":"ok","disableAt":36000,"warnAt":28800},"perCard":{"status":"ok","disableAt":1000,"warnAt":800}},"boards":{"totalMembersPerBoard":{"status":"ok","disableAt":1600,"warnAt":1280},"totalAccessRequestsPerBoard":{"status":"ok","disableAt":4000,"warnAt":3200}},"cards":{"openPerBoard":{"status":"ok","disableAt":5000,"warnAt":4000},"openPerList":{"status":"ok","disableAt":5000,"warnAt":4000},"totalPerBoard":{"status":"ok","disableAt":2000000,"warnAt":1600000},"totalPerList":{"status":"ok","disableAt":1000000,"warnAt":800000}},"checklists":{"perBoard":{"status":"ok","disableAt":1800000,"warnAt":1440000},"perCard":{"status":"ok","disableAt":500,"warnAt":400}},"checkItems":{"perChecklist":{"status":"ok","disableAt":200,"warnAt":160}},"customFields":{"perBoard":{"status":"ok","disableAt":50,"warnAt":40}},"customFieldOptions":{"perField":{"status":"ok","disableAt":50,"warnAt":40}},"labels":{"perBoard":{"status":"ok","disableAt":1000,"warnAt":800}},"lists":{"openPerBoard":{"status":"ok","disableAt":500,"warnAt":400},"totalPerBoard":{"status":"ok","disableAt":3000,"warnAt":2400}},"stickers":{"perCard":{"status":"ok","disableAt":70,"warnAt":56}},"reactions":{"perAction":{"status":"ok","disableAt":900,"warnAt":720},"uniquePerAction":{"status":"ok","disableAt":17,"warnAt":14}}},"pinned":false,"starred":false,"url":"https://trello.com/b/qpYBekjt/testboardnew","prefs":{"permissionLevel":"org","hideVotes":false,"voting":"disabled","comments":"members","invitations":"members","selfJoin":true,"cardCovers":true,"isTemplate":false,"cardAging":"regular","calendarFeedEnabled":false,"hiddenPluginBoardButtons":[],"switcherViews":[{"_id":"63dee887b0ed3558c3793a3a","viewType":"Board","enabled":true,"typeName":"SwitcherViews","id":"63dee887b0ed3558c3793a3a"},{"_id":"63dee887b0ed3558c3793a3b","viewType":"Table","enabled":true,"typeName":"SwitcherViews","id":"63dee887b0ed3558c3793a3b"},{"_id":"63dee887b0ed3558c3793a3c","viewType":"Calendar","enabled":false,"typeName":"SwitcherViews","id":"63dee887b0ed3558c3793a3c"},{"_id":"63dee887b0ed3558c3793a3d","viewType":"Dashboard","enabled":false,"typeName":"SwitcherViews","id":"63dee887b0ed3558c3793a3d"},{"_id":"63dee887b0ed3558c3793a3e","viewType":"Timeline","enabled":false,"typeName":"SwitcherViews","id":"63dee887b0ed3558c3793a3e"},{"_id":"63dee887b0ed3558c3793a3f","viewType":"Map","enabled":false,"typeName":"SwitcherViews","id":"63dee887b0ed3558c3793a3f"}],"background":"611a97ed741bfb5782acf105","backgroundColor":null,"backgroundImage":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/original/a2614b98135f746b4b1d38a1eef0240c/photo-1629006307992-dc1891987730","backgroundImageScaled":[{"width":67,"height":100,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/67x100/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":128,"height":192,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/128x192/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":320,"height":480,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/320x480/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":640,"height":960,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/640x960/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":683,"height":1024,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/683x1024/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":854,"height":1280,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/854x1280/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":1067,"height":1600,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/1067x1600/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":1281,"height":1920,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/1281x1920/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"},{"width":1366,"height":2048,"url":"https://trello-backgrounds.s3.amazonaws.com/SharedBackground/original/a2614b98135f746b4b1d38a1eef0240c/photo-1629006307992-dc1891987730"}],"backgroundTile":false,"backgroundBrightness":"light","backgroundBottomColor":"#7c8280","backgroundTopColor":"#aeb0ae","canBePublic":true,"canBeEnterprise":true,"canBeOrg":true,"canBePrivate":true,"canInvite":true},"shortLink":"qpYBekjt","subscribed":false,"labelNames":{"green":"","yellow":"","orange":"","red":"","purple":"","blue":"","sky":"","lime":"","pink":"","black":"","green_dark":"","yellow_dark":"","orange_dark":"","red_dark":"","purple_dark":"","blue_dark":"","sky_dark":"","lime_dark":"","pink_dark":"","black_dark":"","green_light":"","yellow_light":"","orange_light":"","red_light":"","purple_light":"","blue_light":"","sky_light":"","lime_light":"","pink_light":"","black_light":""},"powerUps":[],"dateLastActivity":"2023-03-16T20:10:54.595Z","dateLastView":"2023-03-17T11:01:50.283Z","shortUrl":"https://trello.com/b/qpYBekjt","idTags":[],"datePluginDisable":null,"creationMethod":"automatic","ixUpdate":"52","templateGallery":null,"enterpriseOwned":false,"idBoardSource":null,"premiumFeatures":["additionalBoardBackgrounds","additionalStickers","customBoardBackgrounds","customEmoji","customStickers","plugins"],"idMemberCreator":"610be3762899a26d04256dae","memberships":[{"idMember":"610be3762899a26d04256dae","memberType":"admin","unconfirmed":false,"deactivated":false,"id":"611aa586ef5f2c8e1deec8ba"}]},"emitted_at":1679051289979} {"stream":"cards","data":{"id":"6127a3488a027b342502ba23","badges":{"attachmentsByType":{"trello":{"board":0,"card":0}},"location":false,"votes":0,"viewingMemberVoted":false,"subscribed":false,"fogbugz":"","checkItems":0,"checkItemsChecked":0,"checkItemsEarliestDue":null,"comments":0,"attachments":0,"description":false,"due":null,"dueComplete":false,"start":null},"checkItemStates":null,"closed":true,"dueComplete":false,"dateLastActivity":"2021-08-26T14:20:58.508Z","desc":"","descData":null,"due":null,"dueReminder":null,"email":null,"idBoard":"611aa0ef37acd675af67dc9b","idChecklists":[],"idList":"611aa0ef37acd675af67dc9d","idMembers":[],"idMembersVoted":[],"idShort":7,"idAttachmentCover":null,"labels":[],"idLabels":[],"manualCoverAttachment":false,"name":"Demo_test","pos":32767.5,"shortLink":"bbhWatIg","shortUrl":"https://trello.com/c/bbhWatIg","start":null,"subscribed":false,"url":"https://trello.com/c/bbhWatIg/7-demotest","cover":{"idAttachment":null,"color":null,"idUploadedBackground":null,"size":"normal","brightness":"dark","idPlugin":null},"isTemplate":false,"cardRole":null,"pluginData":[],"customFieldItems":[],"members":[]},"emitted_at":1679051290436} @@ -13,3 +10,7 @@ {"stream":"lists","data":{"id":"611aa0ef37acd675af67dc9e","name":"Done","closed":false,"idBoard":"611aa0ef37acd675af67dc9b","pos":49152,"subscribed":false,"softLimit":null},"emitted_at":1679051291731} {"stream":"users","data":{"id":"610be3762899a26d04256dae","fullName":"integration test","username":"integrationtest19"},"emitted_at":1679051292221} {"stream":"users","data":{"id":"610be3762899a26d04256dae","fullName":"integration test","username":"integrationtest19"},"emitted_at":1679051292287} +{"stream":"organizations","data":{"id":"610be50a0537086c571a5684","creationMethod":null,"name":"airbyteworkspace","credits":[],"displayName":"Airbyte Workspace","desc":"","descData":{"emoji":{}},"domainName":"airbyte.io","idBoards":["611aa0ef37acd675af67dc9b","611aa586ef5f2c8e1deec8b6"],"idEnterprise":null,"idMemberCreator":"610be3762899a26d04256dae","invited":false,"invitations":[],"limits":{"orgs":{"totalMembersPerOrg":{"status":"ok","disableAt":4000,"warnAt":3200},"freeBoardsPerOrg":{"status":"ok","disableAt":10,"warnAt":3}}},"memberships":[{"idMember":"610be3762899a26d04256dae","memberType":"admin","unconfirmed":false,"deactivated":false,"id":"610be50a0537086c571a5685"}],"membersCount":1,"prefs":{"permissionLevel":"private","orgInviteRestrict":[],"boardInviteRestrict":"any","externalMembersDisabled":false,"associatedDomain":null,"googleAppsVersion":1,"boardVisibilityRestrict":{"private":"org","org":"org","enterprise":"org","public":"org"},"boardDeleteRestrict":{"private":"org","org":"org","enterprise":"org","public":"org"},"attachmentRestrictions":null},"powerUps":[],"products":[],"billableMemberCount":1,"activeBillableMemberCount":1,"billableCollaboratorCount":0,"url":"https://trello.com/w/airbyteworkspace","website":null,"logoHash":null,"logoUrl":null,"premiumFeatures":["additionalBoardBackgrounds","additionalStickers","customBoardBackgrounds","customEmoji","customStickers","plugins"],"promotions":[],"enterpriseJoinRequest":{},"standardVariation":null,"availableLicenseCount":null,"maximumLicenseCount":null,"ixUpdate":"6","teamType":null},"emitted_at":1679382416305} +{"stream":"actions","data":{"id":"6413643417610bdd29d36c47","idMemberCreator":"610be3762899a26d04256dae","data":{"card":{"name":"test_1 (fixed)","id":"611aa1eae05ae943ada67f8a","idShort":3,"shortLink":"FhvCm05w"},"old":{"name":"test_1"},"board":{"id":"611aa0ef37acd675af67dc9b","name":"test_board","shortLink":"Lj11WRiJ"},"list":{"id":"611aa0ef37acd675af67dc9c","name":"To Do"}},"appCreator":null,"type":"updateCard","date":"2023-03-16T18:47:16.404Z","limits":null,"memberCreator":{"id":"610be3762899a26d04256dae","activityBlocked":false,"avatarHash":"eaed38443ff1d12aa4ce2c5d03faafd1","avatarUrl":"https://trello-members.s3.amazonaws.com/610be3762899a26d04256dae/eaed38443ff1d12aa4ce2c5d03faafd1","fullName":"integration test","idMemberReferrer":null,"initials":"IT","nonPublic":{},"nonPublicAvailable":true,"username":"integrationtest19"}},"emitted_at":1679393622531} +{"stream":"actions","data":{"id":"6127a34a824394333c4b089e","idMemberCreator":"610be3762899a26d04256dae","data":{"old":{"closed":false},"card":{"closed":true,"id":"6127a3488a027b342502ba23","name":"Demo_test","idShort":7,"shortLink":"bbhWatIg"},"board":{"id":"611aa0ef37acd675af67dc9b","name":"test_board","shortLink":"Lj11WRiJ"},"list":{"id":"611aa0ef37acd675af67dc9d","name":"Doing"}},"appCreator":{"id":"56ff07afbc287582d36e19ac","name":"Butler","icon":{"url":"https://app.butlerfortrello.com/assets/icon-butler-bot.png"}},"type":"updateCard","date":"2021-08-26T14:20:58.523Z","limits":null,"memberCreator":{"id":"610be3762899a26d04256dae","activityBlocked":false,"avatarHash":"eaed38443ff1d12aa4ce2c5d03faafd1","avatarUrl":"https://trello-members.s3.amazonaws.com/610be3762899a26d04256dae/eaed38443ff1d12aa4ce2c5d03faafd1","fullName":"integration test","idMemberReferrer":null,"initials":"IT","nonPublic":{},"nonPublicAvailable":true,"username":"integrationtest19"}},"emitted_at":1679393622532} +{"stream":"actions","data":{"id":"6127a34ac4dc0b1d29f7667f","idMemberCreator":"610be3762899a26d04256dae","data":{"old":{"pos":131071},"card":{"pos":32767.5,"id":"6127a3488a027b342502ba23","name":"Demo_test","idShort":7,"shortLink":"bbhWatIg"},"board":{"id":"611aa0ef37acd675af67dc9b","name":"test_board","shortLink":"Lj11WRiJ"},"list":{"id":"611aa0ef37acd675af67dc9d","name":"Doing"}},"appCreator":{"id":"56ff07afbc287582d36e19ac","name":"Butler","icon":{"url":"https://app.butlerfortrello.com/assets/icon-butler-bot.png"}},"type":"updateCard","date":"2021-08-26T14:20:58.193Z","limits":null,"memberCreator":{"id":"610be3762899a26d04256dae","activityBlocked":false,"avatarHash":"eaed38443ff1d12aa4ce2c5d03faafd1","avatarUrl":"https://trello-members.s3.amazonaws.com/610be3762899a26d04256dae/eaed38443ff1d12aa4ce2c5d03faafd1","fullName":"integration test","idMemberReferrer":null,"initials":"IT","nonPublic":{},"nonPublicAvailable":true,"username":"integrationtest19"}},"emitted_at":1679393622532} diff --git a/airbyte-integrations/connectors/source-trello/source_trello/schemas/organizations.json b/airbyte-integrations/connectors/source-trello/source_trello/schemas/organizations.json new file mode 100644 index 0000000000000..98f9c73207f66 --- /dev/null +++ b/airbyte-integrations/connectors/source-trello/source_trello/schemas/organizations.json @@ -0,0 +1,198 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "credits": { + "type": "array" + }, + "displayName": { + "type": ["null", "string"] + }, + "desc": { + "type": ["null", "string"] + }, + "descData": { + "type": "object", + "properties": { + "emoji": { + "type": "object" + } + } + }, + "domainName": { + "type": ["null", "string"] + }, + "idBoards": { + "type": "array", + "items": { + "type": ["null", "string"] + } + }, + "idMemberCreator": { + "type": ["null", "string"] + }, + "invited": { + "type": ["null", "boolean"] + }, + "invitations": { + "type": "array" + }, + "limits": { + "type": "object", + "properties": { + "orgs": { + "type": "object", + "properties": { + "totalMembersPerOrg": { + "type": "object", + "properties": { + "status": { + "type": ["null", "string"] + }, + "disableAt": { + "type": ["null", "integer"] + }, + "warnAt": { + "type": ["null", "integer"] + } + } + }, + "freeBoardsPerOrg": { + "type": "object", + "properties": { + "status": { + "type": ["null", "string"] + }, + "disableAt": { + "type": ["null", "integer"] + }, + "warnAt": { + "type": ["null", "integer"] + } + } + } + } + } + } + }, + "memberships": { + "type": "array", + "items": { + "type": "object", + "properties": { + "idMember": { + "type": ["null", "string"] + }, + "memberType": { + "type": ["null", "string"] + }, + "unconfirmed": { + "type": ["null", "boolean"] + }, + "deactivated": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "string"] + } + } + } + }, + "membersCount": { + "type": ["null", "integer"] + }, + "prefs": { + "type": "object", + "properties": { + "permissionLevel": { + "type": ["null", "string"] + }, + "orgInviteRestrict": { + "type": "array" + }, + "boardInviteRestrict": { + "type": ["null", "string"] + }, + "externalMembersDisabled": { + "type": ["null", "boolean"] + }, + "googleAppsVersion": { + "type": ["null", "integer"] + }, + "boardVisibilityRestrict": { + "type": "object", + "properties": { + "private": { + "type": ["null", "string"] + }, + "org": { + "type": ["null", "string"] + }, + "enterprise": { + "type": ["null", "string"] + }, + "public": { + "type": ["null", "string"] + } + } + }, + "boardDeleteRestrict": { + "type": "object", + "properties": { + "private": { + "type": ["null", "string"] + }, + "org": { + "type": ["null", "string"] + }, + "enterprise": { + "type": ["null", "string"] + }, + "public": { + "type": ["null", "string"] + } + } + } + } + }, + "powerUps": { + "type": "array" + }, + "products": { + "type": "array" + }, + "billableMemberCount": { + "type": ["null", "integer"] + }, + "activeBillableMemberCount": { + "type": ["null", "integer"] + }, + "billableCollaboratorCount": { + "type": ["null", "integer"] + }, + "url": { + "type": ["null", "string"] + }, + "premiumFeatures": { + "type": "array", + "items": { + "type": ["null", "string"] + } + }, + "promotions": { + "type": "array" + }, + "enterpriseJoinRequest": { + "type": "object" + }, + "ixUpdate": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-trello/source_trello/source.py b/airbyte-integrations/connectors/source-trello/source_trello/source.py index 2d0c7c9edb59a..559937910bce8 100644 --- a/airbyte-integrations/connectors/source-trello/source_trello/source.py +++ b/airbyte-integrations/connectors/source-trello/source_trello/source.py @@ -14,7 +14,7 @@ from .auth import TrelloAuthenticator from .utils import TrelloRequestRateLimits as balancer -from .utils import read_full_refresh +from .utils import read_all_boards class TrelloStream(HttpStream, ABC): @@ -54,11 +54,11 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class ChildStreamMixin: - parent_stream_class: Optional[TrelloStream] = None - def stream_slices(self, sync_mode, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for item in self.parent_stream_class(config=self.config).read_records(sync_mode=sync_mode): - yield {"id": item["id"]} + board_ids = set(self.config.get("board_ids", [])) + for board_id in read_all_boards(Boards(self.config), Organizations(self.config)): + if not board_ids or board_id in board_ids: + yield {"id": board_id} class IncrementalTrelloStream(TrelloStream, ABC): @@ -112,7 +112,6 @@ class Cards(ChildStreamMixin, TrelloStream): Endpoint: https://api.trello.com/1/boards//cards/all """ - parent_stream_class = Boards limit = 500 extra_params = { "customFieldItems": "true", @@ -138,7 +137,6 @@ class Checklists(ChildStreamMixin, TrelloStream): Endpoint: https://api.trello.com/1/boards//checklists """ - parent_stream_class = Boards extra_params = {"fields": "all", "checkItem_fields": "all"} def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -151,20 +149,26 @@ class Lists(ChildStreamMixin, TrelloStream): Endpoint: https://api.trello.com/1/boards//lists """ - parent_stream_class = Boards - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"boards/{stream_slice['id']}/lists" +class Organizations(TrelloStream): + """Return list of all member's organizations + API Docs: https://developer.atlassian.com/cloud/trello/rest/api-group-members/#api-members-id-organizations-get + Endpoint: https://api.trello.com/1/members/me/organizations + """ + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return "members/me/organizations" + + class Users(ChildStreamMixin, TrelloStream): """Return list of all members of a boards. API Docs: https://developer.atlassian.com/cloud/trello/rest/api-group-boards/#api-boards-id-members-get Endpoint: https://api.trello.com/1/boards//members """ - parent_stream_class = Boards - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"boards/{stream_slice['id']}/members" @@ -175,7 +179,6 @@ class Actions(ChildStreamMixin, IncrementalTrelloStream): Endpoint: https://api.trello.com/1/boards//actions """ - parent_stream_class = Boards limit = 1000 def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -202,8 +205,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: config = self._validate_and_transform(config) config["authenticator"] = self._get_authenticator(config) - stream = Boards({**config, "board_ids": []}) - available_boards = {board["id"] for board in read_full_refresh(stream)} + available_boards = set(read_all_boards(Boards({**config, "board_ids": []}), Organizations(config))) unknown_boards = set(config.get("board_ids", [])) - available_boards if unknown_boards: unknown_boards = ", ".join(sorted(unknown_boards)) @@ -212,4 +214,4 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: config["authenticator"] = self._get_authenticator(config) - return [Actions(config), Boards(config), Cards(config), Checklists(config), Lists(config), Users(config)] + return [Actions(config), Boards(config), Cards(config), Checklists(config), Lists(config), Users(config), Organizations(config)] diff --git a/airbyte-integrations/connectors/source-trello/source_trello/utils.py b/airbyte-integrations/connectors/source-trello/source_trello/utils.py index 3517c9c028e63..9e3545574bafb 100644 --- a/airbyte-integrations/connectors/source-trello/source_trello/utils.py +++ b/airbyte-integrations/connectors/source-trello/source_trello/utils.py @@ -85,3 +85,18 @@ def read_full_refresh(stream_instance: Stream): records = stream_instance.read_records(stream_slice=_slice, sync_mode=SyncMode.full_refresh) for record in records: yield record + + +def read_all_boards(stream_boards: Stream, stream_organizations: Stream): + board_ids = set() + + for record in read_full_refresh(stream_boards): + if record["id"] not in board_ids: + board_ids.add(record["id"]) + yield record["id"] + + for record in read_full_refresh(stream_organizations): + for board_id in record["idBoards"]: + if board_id not in board_ids: + board_ids.add(board_id) + yield board_id diff --git a/airbyte-integrations/connectors/source-trello/unit_tests/test_source.py b/airbyte-integrations/connectors/source-trello/unit_tests/test_source.py index 6a584ea1e7c3e..c86f790ba8659 100644 --- a/airbyte-integrations/connectors/source-trello/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-trello/unit_tests/test_source.py @@ -13,7 +13,7 @@ def test_streams(mocker): source = SourceTrello() config_mock = MagicMock() streams = source.streams(config_mock) - expected_streams_number = 6 + expected_streams_number = 7 assert len(streams) == expected_streams_number @@ -35,6 +35,12 @@ def test_check_connection(requests_mock): ], ) + requests_mock.get( + "https://api.trello.com/1/members/me/organizations", + headers=NO_SLEEP_HEADERS, + json=[{"id": "org111111111111111111111", "idBoards": ["b11111111111111111111111", "b22222222222222222222222"]}], + ) + source = SourceTrello() status, error = source.check_connection(logger, config) assert status is True diff --git a/airbyte-integrations/connectors/source-trello/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-trello/unit_tests/test_streams.py index 9f2ce738d2750..9254e616ec061 100644 --- a/airbyte-integrations/connectors/source-trello/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-trello/unit_tests/test_streams.py @@ -66,6 +66,12 @@ def test_cards_stream(requests_mock): json=[{"id": "b11111111111111111111111", "name": "board_1"}, {"id": "b22222222222222222222222", "name": "board_2"}], ) + mock_organizations_request = requests_mock.get( + "https://api.trello.com/1/members/me/organizations", + headers=NO_SLEEP_HEADERS, + json=[{"id": "org111111111111111111111", "idBoards": ["b11111111111111111111111", "b22222222222222222222222"]}], + ) + json_responses1 = cycle([ [{"id": "c11111111111111111111111", "name": "card_1"}, {"id": "c22222222222222222222222", "name": "card_2"}], [], @@ -107,5 +113,6 @@ def test_cards_stream(requests_mock): assert records == [] assert mock_boards_request.call_count == 3 + assert mock_organizations_request.call_count == 3 assert mock_cards_request_1.call_count == 2 assert mock_cards_request_2.call_count == 4 diff --git a/connectors.md b/connectors.md index 7009703ea7329..145425664685d 100644 --- a/connectors.md +++ b/connectors.md @@ -239,7 +239,7 @@ | **TikTok Marketing** | TikTok Marketing icon | Source | airbyte/source-tiktok-marketing:2.0.4 | generally_available | [link](https://docs.airbyte.com/integrations/sources/tiktok-marketing) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tiktok-marketing) | `4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35` | | **Timely** | Timely icon | Source | airbyte/source-timely:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/timely) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-timely) | `bc617b5f-1b9e-4a2d-bebe-782fd454a771` | | **Toggl** | Toggl icon | Source | airbyte/source-toggl:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/toggl) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-toggl) | `7e7c844f-2300-4342-b7d3-6dd7992593cd` | -| **Trello** | Trello icon | Source | airbyte/source-trello:0.3.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/trello) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-trello) | `8da67652-004c-11ec-9a03-0242ac130003` | +| **Trello** | Trello icon | Source | airbyte/source-trello:0.3.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/trello) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-trello) | `8da67652-004c-11ec-9a03-0242ac130003` | | **TrustPilot** | TrustPilot icon | Source | airbyte/source-trustpilot:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/trustpilot) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-trustpilot) | `d7e23ea6-d741-4314-9209-a33c91a2e945` | | **Twilio** | Twilio icon | Source | airbyte/source-twilio:0.1.16 | generally_available | [link](https://docs.airbyte.com/integrations/sources/twilio) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-twilio) | `b9dc6155-672e-42ea-b10d-9f1f1fb95ab1` | | **Twilio Taskrouter** | Twilio Taskrouter icon | Source | airbyte/source-twilio-taskrouter:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/twilio-taskrouter) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-twilio-taskrouter) | `2446953b-b794-429b-a9b3-c821ba992a48` | diff --git a/docs/integrations/sources/trello.md b/docs/integrations/sources/trello.md index 1819f3b3bc9e2..e396daef7f168 100644 --- a/docs/integrations/sources/trello.md +++ b/docs/integrations/sources/trello.md @@ -64,6 +64,7 @@ This connector outputs the following streams: * [Checklists](https://developer.atlassian.com/cloud/trello/rest/api-group-boards/#api-boards-id-checklists-get) \(Full Refresh\) * [Lists](https://developer.atlassian.com/cloud/trello/rest/api-group-boards/#api-boards-id-lists-get) \(Full Refresh\) * [Users](https://developer.atlassian.com/cloud/trello/rest/api-group-boards/#api-boards-id-members-get) \(Full Refresh\) + * [Organizations](https://developer.atlassian.com/cloud/trello/rest/api-group-members/#api-members-id-organizations-get) \(Full Refresh\) ### Performance considerations @@ -75,6 +76,7 @@ The Trello connector should not run into Trello API limitations under normal usa | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.3.1 | 2023-03-21 | [24266](https://github.com/airbytehq/airbyte/pull/24266) | Get board ids also from organizations | | 0.3.0 | 2023-03-17 | [24141](https://github.com/airbytehq/airbyte/pull/24141) | Certify to Beta | | 0.2.0 | 2023-03-15 | [24045](https://github.com/airbytehq/airbyte/pull/24045) | Fix schema for boards and cards streams | | 0.1.6 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Updated fields in source-connector specifications |