diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 7c5c09fefeaa7..2a5a75ca2e82b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -583,7 +583,7 @@ - name: Marketo sourceDefinitionId: 9e0556f4-69df-4522-a3fb-03264d36b348 dockerRepository: airbyte/source-marketo - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/sources/marketo icon: marketo.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index c1e3d75e478ff..18796715fdcf3 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5182,7 +5182,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-marketo:0.1.5" +- dockerImage: "airbyte/source-marketo:0.1.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/marketo" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-marketo/Dockerfile b/airbyte-integrations/connectors/source-marketo/Dockerfile index 24f09c8bd1416..cc570dd60e8ff 100644 --- a/airbyte-integrations/connectors/source-marketo/Dockerfile +++ b/airbyte-integrations/connectors/source-marketo/Dockerfile @@ -34,5 +34,5 @@ COPY source_marketo ./source_marketo ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/source-marketo diff --git a/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml index 764901dcc48c2..6a2ba4401610a 100644 --- a/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml @@ -14,16 +14,16 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["activities_visit_webpage"] - timeout_seconds: 4800 + empty_streams: ["lists", "campaigns"] + timeout_seconds: 3600 expect_records: path: "integration_tests/expected_records.txt" incremental: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" future_state_path: "integration_tests/abnormal_state.json" - timeout_seconds: 4800 + timeout_seconds: 3600 full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 4800 + timeout_seconds: 3600 diff --git a/airbyte-integrations/connectors/source-marketo/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-marketo/integration_tests/expected_records.txt index 81f2152caff29..594941380aac2 100644 --- a/airbyte-integrations/connectors/source-marketo/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-marketo/integration_tests/expected_records.txt @@ -1,41 +1,63 @@ -{"stream": "programs", "data": {"id": 1016, "name": "123", "description": "", "createdAt": "2021-09-01T16:02:30Z", "updatedAt": "2022-06-21T06:50:32Z", "url": "https://app-sj32.marketo.com/#EBP1016A1", "type": "Email", "channel": "Email Send", "folder": {"type": "Program", "value": 1003, "folderName": "API Test Program"}, "status": "locked", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476224} -{"stream": "programs", "data": {"id": 1017, "name": "air", "description": "", "createdAt": "2021-09-01T16:09:23Z", "updatedAt": "2022-06-21T06:51:01Z", "url": "https://app-sj32.marketo.com/#EBP1017A1", "type": "Email", "channel": "Email Send", "folder": {"type": "Program", "value": 1003, "folderName": "API Test Program"}, "status": "locked", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476226} -{"stream": "programs", "data": {"id": 1003, "name": "API Test Program", "description": "Sample API Program", "createdAt": "2021-01-18T13:55:44Z", "updatedAt": "2022-06-21T06:54:59Z", "url": "https://app-sj32.marketo.com/#PG1003A1", "type": "Default", "channel": "Email Blast", "folder": {"type": "Folder", "value": 45, "folderName": "Active Marketing Programs"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476226} -{"stream": "programs", "data": {"id": 1018, "name": "Jean Lafleur", "description": "", "createdAt": "2021-09-08T12:49:49Z", "updatedAt": "2022-06-21T06:53:28Z", "url": "https://app-sj32.marketo.com/#PG1018A1", "type": "Default", "channel": "Online Advertising", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476227} -{"stream": "programs", "data": {"id": 1019, "name": "Test", "description": "", "createdAt": "2021-09-08T12:59:25Z", "updatedAt": "2022-06-21T06:53:45Z", "url": "https://app-sj32.marketo.com/#PG1019A1", "type": "Default", "channel": "List Import", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476227} -{"stream": "programs", "data": {"id": 1020, "name": "TEST1", "description": "", "createdAt": "2021-09-08T13:21:41Z", "updatedAt": "2022-06-21T06:54:03Z", "url": "https://app-sj32.marketo.com/#PG1020A1", "type": "Default", "channel": "Operational", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476227} -{"stream": "programs", "data": {"id": 1021, "name": "TEST_23", "description": "This is for Test", "createdAt": "2021-09-09T09:00:21Z", "updatedAt": "2022-06-21T06:54:16Z", "url": "https://app-sj32.marketo.com/#PG1021A1", "type": "Default", "channel": "Web Content", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476227} -{"stream": "programs", "data": {"id": 1022, "name": "Test_Demo1", "description": "bla bla", "createdAt": "2021-09-09T14:40:14Z", "updatedAt": "2022-06-21T06:54:29Z", "url": "https://app-sj32.marketo.com/#PG1022A1", "type": "Default", "channel": "Web Request", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476227} -{"stream": "campaigns", "data": {"id": 1019, "name": "Form Smart Campaign", "type": "trigger", "programName": "Form Program", "programId": 1002, "workspaceName": "Default", "createdAt": "2020-04-09T20:18:24Z", "updatedAt": "2020-10-22T09:03:44Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "campaigns", "data": {"id": 1020, "name": "Smart Campaign number 02", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:37:24Z", "updatedAt": "2021-01-19T22:50:17Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "campaigns", "data": {"id": 1021, "name": "Smart Campaign 03", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:38:53Z", "updatedAt": "2021-01-18T13:38:53Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "campaigns", "data": {"id": 1022, "name": "Smart Campaign Number 1", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:41Z", "updatedAt": "2021-01-18T13:48:41Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "campaigns", "data": {"id": 1023, "name": "Smart Campaign Number 2", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:42Z", "updatedAt": "2021-01-18T13:48:42Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "campaigns", "data": {"id": 1024, "name": "Smart Campaign Number 3", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:43Z", "updatedAt": "2021-01-18T13:48:43Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "campaigns", "data": {"id": 1025, "name": "Smart Campaign Number 4", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:44Z", "updatedAt": "2021-01-18T13:48:44Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "campaigns", "data": {"id": 1026, "name": "Smart Campaign Number 5", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:45Z", "updatedAt": "2021-01-18T13:48:45Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "campaigns", "data": {"id": 1027, "name": "Smart Campaign Number 6", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:46Z", "updatedAt": "2021-01-18T13:48:46Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "campaigns", "data": {"id": 1028, "name": "Smart Campaign Number 7", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:47Z", "updatedAt": "2021-01-18T13:48:47Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "campaigns", "data": {"id": 1029, "name": "Smart Campaign Number 8", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:48Z", "updatedAt": "2021-01-18T13:48:48Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "campaigns", "data": {"id": 1030, "name": "Smart Campaign Number 9", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:49Z", "updatedAt": "2021-01-18T13:48:49Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "campaigns", "data": {"id": 1031, "name": "Smart Campaign Number 10", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:50Z", "updatedAt": "2021-01-18T13:48:50Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "lists", "data": {"id": 1001, "name": "Test list", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:27:23Z", "updatedAt": "2022-06-21T06:58:01Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1002, "name": "Test list number 1", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:28:00Z", "updatedAt": "2021-01-19T21:55:54Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1003, "name": "Test list number 2", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:28:07Z", "updatedAt": "2021-01-19T20:28:09Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1004, "name": "Test list number 3", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:28:13Z", "updatedAt": "2021-01-19T20:28:15Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1005, "name": "Test list number 4", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:28:20Z", "updatedAt": "2021-01-19T20:28:21Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1006, "name": "Test list number 5", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:28:27Z", "updatedAt": "2021-01-19T20:28:28Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1007, "name": "Test list number 6", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:28:34Z", "updatedAt": "2021-01-19T20:28:35Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1008, "name": "Test list number 7", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:28:55Z", "updatedAt": "2021-01-19T20:28:56Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1009, "name": "Test list number 8", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:29:02Z", "updatedAt": "2021-01-19T20:29:03Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1010, "name": "Test list number 9", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:29:09Z", "updatedAt": "2021-01-19T20:29:11Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1011, "name": "Test list number 10", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:29:16Z", "updatedAt": "2021-01-19T20:29:18Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1012, "name": "airbyte", "programName": "EM - Auteur - v1", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-09-02T09:30:58Z", "updatedAt": "2021-09-02T09:30:59Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1012, "name": "airbyte", "programName": "EM - Auteur - v1", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-09-02T09:30:58Z", "updatedAt": "2021-09-02T09:30:59Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1012, "name": "airbyte", "programName": "EM - Auteur - v1", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-09-02T09:30:58Z", "updatedAt": "2021-09-02T09:30:59Z"}, "emitted_at": 1638527852000} -{"stream": "lists", "data": {"id": 1012, "name": "airbyte", "programName": "EM - Auteur - v1", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-09-02T09:30:58Z", "updatedAt": "2021-09-02T09:30:59Z"}, "emitted_at": 1638527853000} -{"stream": "leads", "data": {"company": "Airbyte", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 876, "mktoName": "Expecto Patronum", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Expecto", "middleName": null, "lastName": "Patronum", "email": "expecto@patronum.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "84", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T07:49:25Z", "updatedAt": "2022-06-21T07:50:05Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "876", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "876", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1655800613397} -{"stream": "leads", "data": {"company": "FedEx", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 877, "mktoName": "Frodo Baggins", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Frodo", "middleName": null, "lastName": "Baggins", "email": "frodo@baggins.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "85", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T08:30:55Z", "updatedAt": "2022-06-21T08:30:55Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "877", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "877", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1655800613399} -{"stream": "leads", "data": {"company": "PizzaHouse", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 878, "mktoName": "Peter Petegrew", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Peter", "middleName": null, "lastName": "Petegrew", "email": "peter@petegrew.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "86", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T08:31:42Z", "updatedAt": "2022-06-21T08:31:42Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "878", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "878", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1655800613400} -{"stream": "leads", "data": {"company": "SportLife", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 879, "mktoName": "Dudley Dursley", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Dudley", "middleName": null, "lastName": "Dursley", "email": "dudley@dursley.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "87", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T08:32:37Z", "updatedAt": "2022-06-21T08:32:37Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "879", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "879", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1655800613400} -{"stream": "leads", "data": {"company": "KeenEye", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 880, "mktoName": "Alastor Moody", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Alastor", "middleName": null, "lastName": "Moody", "email": "alastor@moody.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "88", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T08:34:25Z", "updatedAt": "2022-06-21T08:34:25Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "880", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "880", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1655800613401} +{"stream": "programs", "data": {"id": 1003, "name": "API Test Program", "description": "Sample API Program", "createdAt": "2021-01-18T13:55:44Z", "updatedAt": "2022-06-21T06:54:59Z", "url": "https://app-sj32.marketo.com/#PG1003A1", "type": "Default", "channel": "Email Blast", "folder": {"type": "Folder", "value": 45, "folderName": "Active Marketing Programs"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1661177167720} +{"stream": "programs", "data": {"id": 1018, "name": "Jean Lafleur", "description": "", "createdAt": "2021-09-08T12:49:49Z", "updatedAt": "2022-06-21T06:53:28Z", "url": "https://app-sj32.marketo.com/#PG1018A1", "type": "Default", "channel": "Online Advertising", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1661177167723} +{"stream": "programs", "data": {"id": 1019, "name": "Test", "description": "", "createdAt": "2021-09-08T12:59:25Z", "updatedAt": "2022-06-21T06:53:45Z", "url": "https://app-sj32.marketo.com/#PG1019A1", "type": "Default", "channel": "List Import", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1661177167724} +{"stream": "programs", "data": {"id": 1020, "name": "TEST1", "description": "", "createdAt": "2021-09-08T13:21:41Z", "updatedAt": "2022-06-21T06:54:03Z", "url": "https://app-sj32.marketo.com/#PG1020A1", "type": "Default", "channel": "Operational", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1661177167725} +{"stream": "programs", "data": {"id": 1021, "name": "TEST_23", "description": "This is for Test", "createdAt": "2021-09-09T09:00:21Z", "updatedAt": "2022-06-21T06:54:16Z", "url": "https://app-sj32.marketo.com/#PG1021A1", "type": "Default", "channel": "Web Content", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1661177167725} +{"stream": "programs", "data": {"id": 1022, "name": "Test_Demo1", "description": "bla bla", "createdAt": "2021-09-09T14:40:14Z", "updatedAt": "2022-06-21T06:54:29Z", "url": "https://app-sj32.marketo.com/#PG1022A1", "type": "Default", "channel": "Web Request", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1661177167726} +{"stream": "leads", "data": {"company": "Airbyte", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 876, "mktoName": "Expecto Patronum", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Expecto", "middleName": null, "lastName": "Patronum", "email": "expecto@patronum.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "84", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T07:49:25Z", "updatedAt": "2022-06-21T07:50:05Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "876", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "876", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1661177364549} +{"stream": "leads", "data": {"company": "FedEx", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 877, "mktoName": "Frodo Baggins", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Frodo", "middleName": null, "lastName": "Baggins", "email": "frodo@baggins.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "85", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T08:30:55Z", "updatedAt": "2022-06-21T08:30:55Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "877", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "877", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1661177364566} +{"stream": "leads", "data": {"company": "PizzaHouse", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 878, "mktoName": "Peter Petegrew", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Peter", "middleName": null, "lastName": "Petegrew", "email": "peter@petegrew.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "86", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T08:31:42Z", "updatedAt": "2022-06-21T08:31:42Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "878", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "878", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1661177364568} +{"stream": "leads", "data": {"company": "SportLife", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 879, "mktoName": "Dudley Dursley", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Dudley", "middleName": null, "lastName": "Dursley", "email": "dudley@dursley.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "87", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T08:32:37Z", "updatedAt": "2022-06-21T08:32:37Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "879", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "879", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1661177364570} +{"stream": "leads", "data": {"company": "KeenEye", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 880, "mktoName": "Alastor Moody", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Alastor", "middleName": null, "lastName": "Moody", "email": "alastor@moody.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "88", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T08:34:25Z", "updatedAt": "2022-06-21T08:34:25Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "880", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "880", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1661177364572} +{"stream": "activity_types", "data": {"id": 1, "name": "Visit Webpage", "description": "User visits a web page", "primaryAttribute": {"name": "Webpage ID", "dataType": "integer"}, "attributes": [{"name": "Client IP Address", "dataType": "string"}, {"name": "Query Parameters", "dataType": "string"}, {"name": "Referrer URL", "dataType": "string"}, {"name": "Search Engine", "dataType": "string"}, {"name": "Search Query", "dataType": "string"}, {"name": "User Agent", "dataType": "string"}, {"name": "Webpage URL", "dataType": "string"}]}, "emitted_at": 1661177367593} +{"stream": "activity_types", "data": {"id": 2, "name": "Fill Out Form", "description": "User fills out and submits a form on web page", "primaryAttribute": {"name": "Webform ID", "dataType": "integer"}, "attributes": [{"name": "Client IP Address", "dataType": "string"}, {"name": "Form Fields", "dataType": "text"}, {"name": "Query Parameters", "dataType": "string"}, {"name": "Referrer URL", "dataType": "string"}, {"name": "User Agent", "dataType": "string"}, {"name": "Webpage ID", "dataType": "integer"}]}, "emitted_at": 1661177367603} +{"stream": "activity_types", "data": {"id": 3, "name": "Click Link", "description": "User clicks link on a page", "primaryAttribute": {"name": "Link ID", "dataType": "integer"}, "attributes": [{"name": "Client IP Address", "dataType": "string"}, {"name": "Query Parameters", "dataType": "string"}, {"name": "Referrer URL", "dataType": "string"}, {"name": "User Agent", "dataType": "string"}, {"name": "Webpage ID", "dataType": "integer"}]}, "emitted_at": 1661177367603} +{"stream": "activity_types", "data": {"id": 6, "name": "Send Email", "description": "Send Marketo Email to a person", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Choice Number", "dataType": "integer"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Step ID", "dataType": "integer"}, {"name": "Test Variant", "dataType": "integer"}]}, "emitted_at": 1661177367603} +{"stream": "activity_types", "data": {"id": 7, "name": "Email Delivered", "description": "Marketo Email is delivered to a lead/contact", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Choice Number", "dataType": "integer"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Step ID", "dataType": "integer"}, {"name": "Test Variant", "dataType": "integer"}]}, "emitted_at": 1661177367603} +{"stream": "activity_types", "data": {"id": 8, "name": "Email Bounced", "description": "Marketo Email is bounced for a lead", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Category", "dataType": "string"}, {"name": "Choice Number", "dataType": "integer"}, {"name": "Details", "dataType": "string"}, {"name": "Email", "dataType": "string"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Step ID", "dataType": "integer"}, {"name": "Subcategory", "dataType": "string"}, {"name": "Test Variant", "dataType": "integer"}]}, "emitted_at": 1661177367603} +{"stream": "activity_types", "data": {"id": 9, "name": "Unsubscribe Email", "description": "Person unsubscribed from Marketo Emails", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "Integer"}, {"name": "Client IP Address", "dataType": "string"}, {"name": "Form Fields", "dataType": "text"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Query Parameters", "dataType": "string"}, {"name": "Referrer URL", "dataType": "string"}, {"name": "Test Variant", "dataType": "integer"}, {"name": "User Agent", "dataType": "string"}, {"name": "Webform ID", "dataType": "integer"}, {"name": "Webpage ID", "dataType": "integer"}]}, "emitted_at": 1661177367604} +{"stream": "activity_types", "data": {"id": 10, "name": "Open Email", "description": "User opens Marketo Email", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Bot Activity Pattern", "dataType": "string"}, {"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Choice Number", "dataType": "integer"}, {"name": "Device", "dataType": "string"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Is Bot Activity", "dataType": "boolean"}, {"name": "Is Mobile Device", "dataType": "boolean"}, {"name": "Platform", "dataType": "string"}, {"name": "Step ID", "dataType": "integer"}, {"name": "Test Variant", "dataType": "integer"}, {"name": "User Agent", "dataType": "string"}]}, "emitted_at": 1661177367604} +{"stream": "activity_types", "data": {"id": 11, "name": "Click Email", "description": "User clicks on a link in a Marketo Email", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Bot Activity Pattern", "dataType": "string"}, {"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Choice Number", "dataType": "integer"}, {"name": "Device", "dataType": "string"}, {"name": "Is Bot Activity", "dataType": "boolean"}, {"name": "Is Mobile Device", "dataType": "boolean"}, {"name": "Is Predictive", "dataType": "boolean"}, {"name": "Link", "dataType": "string"}, {"name": "Link ID", "dataType": "string"}, {"name": "Platform", "dataType": "string"}, {"name": "Step ID", "dataType": "integer"}, {"name": "Test Variant", "dataType": "integer"}, {"name": "User Agent", "dataType": "string"}]}, "emitted_at": 1661177367604} +{"stream": "activity_types", "data": {"id": 12, "name": "New Lead", "description": "New person/record is added to the lead database", "attributes": [{"name": "Created Date", "dataType": "date"}, {"name": "Form Name", "dataType": "string"}, {"name": "Lead Source", "dataType": "string"}, {"name": "List Name", "dataType": "string"}, {"name": "SFDC Type", "dataType": "string"}, {"name": "Source Type", "dataType": "string"}, {"name": "API Method Name", "dataType": "string"}, {"name": "Modifying User", "dataType": "string"}, {"name": "Request Id", "dataType": "string"}]}, "emitted_at": 1661177367604} +{"stream": "activity_types", "data": {"id": 13, "name": "Change Data Value", "description": "Changed attribute value for a person/record", "primaryAttribute": {"name": "Attribute Name", "dataType": "integer"}, "attributes": [{"name": "New Value", "dataType": "string"}, {"name": "Old Value", "dataType": "string"}, {"name": "Reason", "dataType": "string"}, {"name": "Source", "dataType": "string"}, {"name": "API Method Name", "dataType": "string"}, {"name": "Modifying User", "dataType": "string"}, {"name": "Request Id", "dataType": "string"}]}, "emitted_at": 1661177367604} +{"stream": "activity_types", "data": {"id": 22, "name": "Change Score", "description": "Modify the value of a score field", "primaryAttribute": {"name": "Score Name", "dataType": "integer"}, "attributes": [{"name": "Change Value", "dataType": "string"}, {"name": "New Value", "dataType": "integer"}, {"name": "Old Value", "dataType": "integer"}, {"name": "Reason", "dataType": "string"}]}, "emitted_at": 1661177367604} +{"stream": "activity_types", "data": {"id": 24, "name": "Add to List", "description": "Add a person/record to a list", "primaryAttribute": {"name": "List ID", "dataType": "integer"}, "attributes": [{"name": "Source", "dataType": "string"}]}, "emitted_at": 1661177367605} +{"stream": "activity_types", "data": {"id": 25, "name": "Remove from List", "description": "Remove this lead from a list", "primaryAttribute": {"name": "List ID", "dataType": "integer"}, "attributes": [{"name": "Source", "dataType": "string"}]}, "emitted_at": 1661177367605} +{"stream": "activity_types", "data": {"id": 27, "name": "Email Bounced Soft", "description": "Campaign Email is bounced soft for a lead", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Category", "dataType": "string"}, {"name": "Choice Number", "dataType": "integer"}, {"name": "Details", "dataType": "string"}, {"name": "Email", "dataType": "string"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Step ID", "dataType": "integer"}, {"name": "Subcategory", "dataType": "string"}, {"name": "Test Variant", "dataType": "integer"}]}, "emitted_at": 1661177367605} +{"stream": "activity_types", "data": {"id": 32, "name": "Merge Leads", "description": "Merge two or more leads into a single record", "primaryAttribute": {"name": "Lead ID", "dataType": "integer"}, "attributes": [{"name": "Merge IDs", "dataType": "array"}, {"name": "Master Updated", "dataType": "boolean"}, {"name": "Merged in Sales", "dataType": "boolean"}, {"name": "Merge Source", "dataType": "string"}, {"name": "API Method Name", "dataType": "string"}, {"name": "Modifying User", "dataType": "string"}, {"name": "Request Id", "dataType": "string"}]}, "emitted_at": 1661177367605} +{"stream": "activity_types", "data": {"id": 34, "name": "Add to Opportunity", "description": "Add to an Opportunity", "primaryAttribute": {"name": "Oppty ID", "dataType": "integer"}, "attributes": [{"name": "Is Primary", "dataType": "boolean"}, {"name": "Role", "dataType": "string"}]}, "emitted_at": 1661177367605} +{"stream": "activity_types", "data": {"id": 35, "name": "Remove from Opportunity", "description": "Remove from an Opportunity", "primaryAttribute": {"name": "Oppty ID", "dataType": "integer"}, "attributes": [{"name": "Is Primary", "dataType": "boolean"}, {"name": "Role", "dataType": "string"}]}, "emitted_at": 1661177367605} +{"stream": "activity_types", "data": {"id": 36, "name": "Update Opportunity", "description": "Update an Opportunity", "primaryAttribute": {"name": "Oppty ID", "dataType": "integer"}, "attributes": [{"name": "Attribute Name", "dataType": "string"}, {"name": "Data Value Changes", "dataType": "complex"}, {"name": "New Value", "dataType": "mixed"}, {"name": "Old Value", "dataType": "mixed"}]}, "emitted_at": 1661177367605} +{"stream": "activity_types", "data": {"id": 37, "name": "Delete Lead", "description": "Delete lead from Marketo database", "primaryAttribute": {"name": "Lead ID", "dataType": "integer"}, "attributes": [{"name": "Remove from CRM", "dataType": "boolean"}, {"name": "API Method Name", "dataType": "string"}, {"name": "Modifying User", "dataType": "string"}, {"name": "Request Id", "dataType": "string"}]}, "emitted_at": 1661177367606} +{"stream": "activity_types", "data": {"id": 38, "name": "Send Alert", "description": "Send alert Email about a lead", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Send To List", "dataType": "email_list"}, {"name": "Send To Owner", "dataType": "string"}, {"name": "Send To Smart List", "dataType": "string"}]}, "emitted_at": 1661177367606} +{"stream": "activity_types", "data": {"id": 39, "name": "Send Sales Email", "description": "Send Email using Sales View (Outlook Plugin)", "primaryAttribute": {"name": "Artifact ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Marketo Sales Person ID", "dataType": "integer"}, {"name": "Sales Campaign Name", "dataType": "string"}, {"name": "Sales Campaign URL", "dataType": "string"}, {"name": "Sales Email URL", "dataType": "string"}, {"name": "Sales Template Name", "dataType": "string"}, {"name": "Sales Template URL", "dataType": "string"}, {"name": "Sent by", "dataType": "string"}, {"name": "Source", "dataType": "string"}, {"name": "Template ID", "dataType": "integer"}]}, "emitted_at": 1661177367606} +{"stream": "activity_types", "data": {"id": 40, "name": "Open Sales Email", "description": "User opens a sales Email", "primaryAttribute": {"name": "Artifact ID", "dataType": "integer"}, "attributes": [{"name": "Bot Activity Pattern", "dataType": "string"}, {"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Is Bot Activity", "dataType": "boolean"}, {"name": "Marketo Sales Person ID", "dataType": "integer"}, {"name": "Sales Campaign Name", "dataType": "string"}, {"name": "Sales Campaign URL", "dataType": "string"}, {"name": "Sales Email URL", "dataType": "string"}, {"name": "Sales Template Name", "dataType": "string"}, {"name": "Sales Template URL", "dataType": "string"}, {"name": "Sent by", "dataType": "string"}, {"name": "Source", "dataType": "string"}, {"name": "Template ID", "dataType": "integer"}]}, "emitted_at": 1661177367606} +{"stream": "activity_types", "data": {"id": 41, "name": "Click Sales Email", "description": "User clicks on a link in a sales Email", "primaryAttribute": {"name": "Artifact ID", "dataType": "integer"}, "attributes": [{"name": "Bot Activity Pattern", "dataType": "string"}, {"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Is Bot Activity", "dataType": "boolean"}, {"name": "Is Predictive", "dataType": "boolean"}, {"name": "Link", "dataType": "string"}, {"name": "Marketo Sales Person ID", "dataType": "integer"}, {"name": "Sales Campaign Name", "dataType": "string"}, {"name": "Sales Campaign URL", "dataType": "string"}, {"name": "Sales Email URL", "dataType": "string"}, {"name": "Sales Template Name", "dataType": "string"}, {"name": "Sales Template URL", "dataType": "string"}, {"name": "Sent by", "dataType": "string"}, {"name": "Source", "dataType": "string"}, {"name": "Template ID", "dataType": "integer"}]}, "emitted_at": 1661177367606} +{"stream": "activity_types", "data": {"id": 45, "name": "Receive Sales Email", "description": "Receive Email using Sales View (Outlook Plugin)", "primaryAttribute": {"name": "Artifact ID", "dataType": "integer"}, "attributes": [{"name": "Received by", "dataType": "string"}, {"name": "Source", "dataType": "string"}]}, "emitted_at": 1661177367607} +{"stream": "activity_types", "data": {"id": 47, "name": "Request Campaign", "description": "Campaign membership is requested for lead", "primaryAttribute": {"name": "Campaign ID", "dataType": "integer"}, "attributes": [{"name": "Source", "dataType": "enum"}]}, "emitted_at": 1661177367607} +{"stream": "activity_types", "data": {"id": 48, "name": "Sales Email Bounced", "description": "Sales Email bounced", "primaryAttribute": {"name": "Artifact ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Category", "dataType": "string"}, {"name": "Details", "dataType": "string"}, {"name": "Email", "dataType": "string"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Marketo Sales Person ID", "dataType": "integer"}, {"name": "Sent by", "dataType": "string"}, {"name": "Source", "dataType": "string"}, {"name": "Subcategory", "dataType": "string"}, {"name": "Template ID", "dataType": "integer"}]}, "emitted_at": 1661177367607} +{"stream": "activity_types", "data": {"id": 100, "name": "Change Lead Partition", "description": "Lead partition is changed for a lead", "primaryAttribute": {"name": "New Partition ID", "dataType": "integer"}, "attributes": [{"name": "Old Partition ID", "dataType": "integer"}, {"name": "Reason", "dataType": "string"}]}, "emitted_at": 1661177367607} +{"stream": "activity_types", "data": {"id": 101, "name": "Change Revenue Stage", "description": "Modify the value of a revenue stage field", "primaryAttribute": {"name": "Model ID", "dataType": "integer"}, "attributes": [{"name": "New Stage ID", "dataType": "integer"}, {"name": "Old Stage ID", "dataType": "integer"}, {"name": "Reason", "dataType": "string"}, {"name": "SLA Expiration", "dataType": "datetime"}, {"name": "New Stage", "dataType": "string"}, {"name": "Old Stage", "dataType": "string"}]}, "emitted_at": 1661177367607} +{"stream": "activity_types", "data": {"id": 102, "name": "Change Revenue Stage Manually", "description": "Revenue stage is changed manually (e.g. Lead Database or Salesforce)"}, "emitted_at": 1661177367607} +{"stream": "activity_types", "data": {"id": 104, "name": "Change Status in Progression", "description": "Change a Status in an program progression", "primaryAttribute": {"name": "Program ID", "dataType": "integer"}, "attributes": [{"name": "Acquired By", "dataType": "boolean"}, {"name": "New Status ID", "dataType": "integer"}, {"name": "Old Status ID", "dataType": "integer"}, {"name": "Program Member ID", "dataType": "integer"}, {"name": "Reached Success Date", "dataType": "datetime"}, {"name": "Reason", "dataType": "string"}, {"name": "Registration Code", "dataType": "string"}, {"name": "Status Reason", "dataType": "string"}, {"name": "Success", "dataType": "boolean"}, {"name": "Webinar URL", "dataType": "string"}, {"name": "New Status", "dataType": "string"}, {"name": "Old Status", "dataType": "string"}]}, "emitted_at": 1661177367608} +{"stream": "activity_types", "data": {"id": 108, "name": "Change Segment", "description": "Change segment (within a segmentation) for a lead", "primaryAttribute": {"name": "Segmentation ID", "dataType": "integer"}, "attributes": [{"name": "New Segment ID", "dataType": "integer"}]}, "emitted_at": 1661177367608} +{"stream": "activity_types", "data": {"id": 110, "name": "Call Webhook", "description": "Call a Webhook", "primaryAttribute": {"name": "Webhook", "dataType": "integer"}, "attributes": [{"name": "Error Type", "dataType": "integer"}, {"name": "Response", "dataType": "string"}]}, "emitted_at": 1661177367608} +{"stream": "activity_types", "data": {"id": 111, "name": "Sent Forward to Friend Email", "description": "User Forwards Email to a person/record", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Choice Number", "dataType": "integer"}, {"name": "Lead ID", "dataType": "string"}, {"name": "Step ID", "dataType": "integer"}]}, "emitted_at": 1661177367608} +{"stream": "activity_types", "data": {"id": 112, "name": "Received Forward to Friend Email", "description": "User receives forwarded Email from friend", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Choice Number", "dataType": "integer"}, {"name": "Lead ID", "dataType": "string"}, {"name": "Step ID", "dataType": "integer"}]}, "emitted_at": 1661177367608} +{"stream": "activity_types", "data": {"id": 113, "name": "Add to Nurture", "description": "Add a lead to a nurture program", "primaryAttribute": {"name": "Program ID", "dataType": "integer"}, "attributes": [{"name": "Track ID", "dataType": "integer"}, {"name": "Track Name", "dataType": "string"}]}, "emitted_at": 1661177367608} +{"stream": "activity_types", "data": {"id": 114, "name": "Change Nurture Track", "description": "Change the nurture track for a lead", "primaryAttribute": {"name": "Program ID", "dataType": "integer"}, "attributes": [{"name": "New Track ID", "dataType": "integer"}, {"name": "Previous Track ID", "dataType": "integer"}, {"name": "Previous Track Name", "dataType": "string"}, {"name": "Track Name", "dataType": "string"}]}, "emitted_at": 1661177367608} +{"stream": "activity_types", "data": {"id": 115, "name": "Change Nurture Cadence", "description": "Change the nurture cadence for a lead", "primaryAttribute": {"name": "Program ID", "dataType": "integer"}, "attributes": [{"name": "New Nurture Cadence", "dataType": "string"}, {"name": "Previous Nurture Cadence", "dataType": "string"}]}, "emitted_at": 1661177367609} +{"stream": "activity_types", "data": {"id": 123, "name": "Change Program Member Data", "description": "Change Program Member Data", "primaryAttribute": {"name": "Program ID", "dataType": "integer"}, "attributes": [{"name": "Attribute Display Name", "dataType": "string"}, {"name": "Attribute Name", "dataType": "integer"}, {"name": "New Value", "dataType": "mixed"}, {"name": "Old Value", "dataType": "mixed"}, {"name": "Reason", "dataType": "string"}, {"name": "Source", "dataType": "string"}]}, "emitted_at": 1661177367609} +{"stream": "activity_types", "data": {"id": 145, "name": "Push Lead to Marketo", "primaryAttribute": {"name": "Program ID", "dataType": "integer"}, "attributes": [{"name": "Reason", "dataType": "string"}, {"name": "Source", "dataType": "string"}, {"name": "API Method Name", "dataType": "string"}, {"name": "Modifying User", "dataType": "string"}, {"name": "Request Id", "dataType": "string"}]}, "emitted_at": 1661177367609} +{"stream": "activity_types", "data": {"id": 155, "name": "Execute Campaign", "description": "Invoke an executable campaign", "primaryAttribute": {"name": "Campaign ID", "dataType": "integer"}, "attributes": [{"name": "Qualified", "dataType": "boolean"}, {"name": "Token Context Campaign ID", "dataType": "integer"}, {"name": "Used Parent Campaign Token Context", "dataType": "boolean"}]}, "emitted_at": 1661177367609} +{"stream": "activity_types", "data": {"id": 157, "name": "Reply to Sales Email", "description": "User replies to a sales email", "primaryAttribute": {"name": "Artifact ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Marketo Sales Person ID", "dataType": "integer"}, {"name": "Sales Campaign Name", "dataType": "string"}, {"name": "Sales Campaign URL", "dataType": "string"}, {"name": "Sales Email URL", "dataType": "string"}, {"name": "Sales Template Name", "dataType": "string"}, {"name": "Sales Template URL", "dataType": "string"}, {"name": "Sent by", "dataType": "string"}, {"name": "Source", "dataType": "string"}, {"name": "Template ID", "dataType": "integer"}]}, "emitted_at": 1661177367609} +{"stream": "activity_types", "data": {"id": 158, "name": "Dialogue Engaged", "description": "Lead engages with Dynamic Chat dialogue", "primaryAttribute": {"name": "Dialogue Id", "dataType": "integer"}, "attributes": [{"name": "Page URL", "dataType": "string"}, {"name": "Status", "dataType": "string"}]}, "emitted_at": 1661177367609} +{"stream": "activity_types", "data": {"id": 159, "name": "Document Interacted With", "description": "Lead interacts with a document in Dynamic Chat dialogue", "primaryAttribute": {"name": "Document Id", "dataType": "integer"}, "attributes": [{"name": "Dialogue Id", "dataType": "integer"}, {"name": "Document Downloaded", "dataType": "boolean"}, {"name": "Document Opened", "dataType": "boolean"}, {"name": "Document URL", "dataType": "string"}, {"name": "Search Terms", "dataType": "string"}]}, "emitted_at": 1661177367610} +{"stream": "activity_types", "data": {"id": 160, "name": "Dialogue Appointment Scheduled", "description": "Lead schedules an appointment in Dynamic Chat dialogue", "primaryAttribute": {"name": "Agent Id", "dataType": "integer"}, "attributes": [{"name": "Dialogue Id", "dataType": "integer"}, {"name": "Page URL", "dataType": "string"}, {"name": "Scheduled For", "dataType": "datetime"}, {"name": "Status", "dataType": "string"}]}, "emitted_at": 1661177367610} +{"stream": "activity_types", "data": {"id": 161, "name": "Dialogue Goal Reached", "description": "Lead reaches a goal in Dynamic Chat dialogue", "primaryAttribute": {"name": "Dialogue Id", "dataType": "integer"}, "attributes": [{"name": "Goal Name", "dataType": "string"}, {"name": "Page URL", "dataType": "string"}]}, "emitted_at": 1661177367610} +{"stream": "activity_types", "data": {"id": 400, "name": "Share Content", "description": "Share Content", "primaryAttribute": {"name": "Social App ID", "dataType": "integer"}, "attributes": [{"name": "Share Message", "dataType": "string"}, {"name": "Social App Type ID", "dataType": "integer"}, {"name": "Social Network", "dataType": "string"}, {"name": "Webpage ID", "dataType": "integer"}]}, "emitted_at": 1661177367610} +{"stream": "activities_visit_webpage", "data": {"marketoGUID": "1952", "leadId": 864, "activityDate": "2022-06-22T02:15:29Z", "activityTypeId": 1, "campaignId": 1051, "primaryAttributeValueId": "1", "primaryAttributeValue": "UnsubscribePage", "webpage_url": "/lp/datalineaedev/UnsubscribePage.html", "client_ip_address": "193.254.197.5", "user_agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", "query_parameters": "mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGFKCnuS27x1GyhwFnd8EcZUeBin9Izd97Fjf1_l_oOdzmzE8qMwNcsnzARW-WUaop8fF9bsAT759OzqwwQoiyAK5ss6rwChiZ2", "referrer_url": "http://602-euo-598.mktoweb.com/"}, "emitted_at": 1661177561089} +{"stream": "activities_visit_webpage", "data": {"marketoGUID": "1953", "leadId": 864, "activityDate": "2022-06-22T02:15:27Z", "activityTypeId": 1, "campaignId": 1051, "primaryAttributeValueId": "1", "primaryAttributeValue": "UnsubscribePage", "webpage_url": "/lp/datalineaedev/UnsubscribePage.html", "client_ip_address": "24.154.57.72", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", "query_parameters": "mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGFKCnuS27x1GyhwFnd8EcZUeBin9Izd97Fjf1_l_oOdzmzE8qMwNcsnzARW-WUaop8fF9bsAT759OzqwwQoiyAK5ss6rwChiZ2", "referrer_url": "http://602-euo-598.mktoweb.com/"}, "emitted_at": 1661177561091} +{"stream": "activities_visit_webpage", "data": {"marketoGUID": "2010", "leadId": 104, "activityDate": "2022-06-22T04:33:54Z", "activityTypeId": 1, "campaignId": 1050, "primaryAttributeValueId": "1", "primaryAttributeValue": "UnsubscribePage", "webpage_url": "/lp/datalineaedev/UnsubscribePage.html", "client_ip_address": "73.92.33.56", "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/102.0.5005.87 Mobile/15E148 Safari/604.1", "query_parameters": "mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGFKCnqhn-Lmk6x4LkqZAcT7qEP0iguRDhjSVP0wJlwugOOpeoihjLcCbUUuBeiuzZVR6QkuJhP6A5_w2VjUK9sBq3TqnjZBati", "referrer_url": "http://602-euo-598.mktoweb.com/"}, "emitted_at": 1661177561093} +{"stream": "activities_visit_webpage", "data": {"marketoGUID": "2011", "leadId": 104, "activityDate": "2022-06-22T04:34:00Z", "activityTypeId": 1, "campaignId": null, "primaryAttributeValueId": "2", "primaryAttributeValue": "UnsubscribeConfirm", "webpage_url": "/lp/602-EUO-598/UnsubscribeConfirm.html", "client_ip_address": "73.92.33.56", "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/102.0.5005.87 Mobile/15E148 Safari/604.1", "query_parameters": "aliId=eyJpIjoic1JzaitlMGZrN0Z6M1BoSSIsInQiOiJQcWk3QzFoTGdKUG9iUFpZT3I4WVwvQT09In0%3D", "referrer_url": "http://602-euo-598.mktoweb.com/lp/datalineaedev/UnsubscribePage.html?mkt_unsubscribe=1"}, "emitted_at": 1661177561094} +{"stream": "activities_visit_webpage", "data": {"marketoGUID": "2014", "leadId": 864, "activityDate": "2022-06-22T06:53:42Z", "activityTypeId": 1, "campaignId": 1050, "primaryAttributeValueId": "1", "primaryAttributeValue": "UnsubscribePage", "webpage_url": "/lp/datalineaedev/UnsubscribePage.html", "client_ip_address": "41.226.21.166", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", "query_parameters": "mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGFKCnqaXQL6xkCtEfmzW_QXyH_0F2pPytjaVUu6Dj4O3t551YEee-hBHahI_1f7Ls0RqsqJMQJCimfQaMpv3f4QoE-D97h6NJK", "referrer_url": "http://602-euo-598.mktoweb.com/"}, "emitted_at": 1661177561096} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-marketo/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-marketo/integration_tests/incremental_catalog.json new file mode 100644 index 0000000000000..9a569ed6353af --- /dev/null +++ b/airbyte-integrations/connectors/source-marketo/integration_tests/incremental_catalog.json @@ -0,0 +1,59 @@ +{ + "streams": [ + { + "stream": { + "name": "programs", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": [] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": [] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "lists", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": [] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "leads", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": [] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "activities_visit_webpage", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": [] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py index 8f623c42c97ce..910f74e1e6906 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py @@ -30,6 +30,8 @@ def __init__(self, config: Mapping[str, Any], stream_name: str = None, param: Ma super().__init__(authenticator=config["authenticator"]) self.config = config self.start_date = config["start_date"] + # this is done for test purposes, the field is not exposed to spec.json! + self.end_date = config.get("end_date") self.window_in_days = config.get("window_in_days", 30) self._url_base = config["domain_url"].rstrip("/") + "/" self.stream_name = stream_name @@ -76,6 +78,10 @@ def normalize_datetime(self, dt: str, format="%Y-%m-%dT%H:%M:%SZ%z"): class IncrementalMarketoStream(MarketoStream): cursor_field = "createdAt" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._state = {} + def filter_by_state(self, stream_state: Mapping[str, Any] = None, record: Mapping[str, Any] = None) -> Iterable: """ Endpoint does not provide query filtering params, but they provide us @@ -83,7 +89,7 @@ def filter_by_state(self, stream_state: Mapping[str, Any] = None, record: Mappin during the parsing. """ - if not stream_state or record[self.cursor_field] >= stream_state.get(self.cursor_field): + if record[self.cursor_field] >= (stream_state or {}).get(self.cursor_field, self.start_date): yield record def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: @@ -92,12 +98,21 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, for record in json_response: yield from self.filter_by_state(stream_state=stream_state, record=record) + @property + def state(self): + return self._state + + @state.setter + def state(self, value): + self._state = value + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - return { + self._state = { self.cursor_field: max( latest_record.get(self.cursor_field, self.start_date), current_stream_state.get(self.cursor_field, self.start_date) ) } + return self._state def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[MutableMapping[str, any]]]: """ @@ -114,16 +129,16 @@ def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwa """ start_date = pendulum.parse(self.start_date) - end_date = pendulum.now() # Determine stream_state, if no stream_state we use start_date if stream_state: start_date = pendulum.parse(stream_state.get(self.cursor_field)) # use the lowest date between start_date and self.end_date, otherwise API fails if start_date is in future - start_date = min(start_date, end_date) + start_date = min(start_date, pendulum.now()) date_slices = [] + end_date = pendulum.parse(self.end_date) if self.end_date else pendulum.now() while start_date <= end_date: # the amount of days for each data-chunk begining from start_date end_date_slice = start_date.add(days=self.window_in_days) @@ -136,6 +151,11 @@ def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwa return date_slices +class SemiIncrementalMarketoStream(IncrementalMarketoStream): + def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[MutableMapping[str, any]]]: + return [None] + + class MarketoExportBase(IncrementalMarketoStream): """ Base class for all the streams which support bulk extract. @@ -459,14 +479,14 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, yield record -class Campaigns(IncrementalMarketoStream): +class Campaigns(SemiIncrementalMarketoStream): """ Return list of all campaigns. API Docs: http://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Campaigns/getCampaignsUsingGET """ -class Lists(IncrementalMarketoStream): +class Lists(SemiIncrementalMarketoStream): """ Return list of all lists. API Docs: http://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Static_Lists/getListsUsingGET diff --git a/docs/integrations/sources/marketo.md b/docs/integrations/sources/marketo.md index 2534405c8d545..34bbb3707121a 100644 --- a/docs/integrations/sources/marketo.md +++ b/docs/integrations/sources/marketo.md @@ -100,11 +100,12 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------| -| `0.1.5` | 2022-08-16 | [15683](https://github.com/airbytehq/airbyte/pull/15683) | Retry failed creation of a job instead of skipping it | -| `0.1.4` | 2022-06-20 | [13930](https://github.com/airbytehq/airbyte/pull/13930) | Process failing creation of export jobs | -| `0.1.3` | 2021-12-10 | [8429](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | -| `0.1.2` | 2021-12-03 | [8483](https://github.com/airbytehq/airbyte/pull/8483) | Improve field conversion to conform schema | -| `0.1.1` | 2021-11-29 | [0000](https://github.com/airbytehq/airbyte/pull/0000) | Fix timestamp value format issue | -| `0.1.0` | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863) | Release Marketo CDK Connector | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------| +| `0.1.6` | 2022-08-21 | [15824](https://github.com/airbytehq/airbyte/pull/15824) | Fix semi incremental streams: do not ignore start date, make one api call instead of multiple | +| `0.1.5` | 2022-08-16 | [15683](https://github.com/airbytehq/airbyte/pull/15683) | Retry failed creation of a job instead of skipping it | +| `0.1.4` | 2022-06-20 | [13930](https://github.com/airbytehq/airbyte/pull/13930) | Process failing creation of export jobs | +| `0.1.3` | 2021-12-10 | [8429](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | +| `0.1.2` | 2021-12-03 | [8483](https://github.com/airbytehq/airbyte/pull/8483) | Improve field conversion to conform schema | +| `0.1.1` | 2021-11-29 | [0000](https://github.com/airbytehq/airbyte/pull/0000) | Fix timestamp value format issue | +| `0.1.0` | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863) | Release Marketo CDK Connector |