diff --git a/airbyte-integrations/connectors/source-harvest/.coveragerc b/airbyte-integrations/connectors/source-harvest/.coveragerc new file mode 100644 index 0000000000000..7abb635210951 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_harvest/run.py diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl index 4b23bf6a6cea2..131e7fe6b2eba 100644 --- a/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl @@ -6,8 +6,8 @@ {"stream": "company", "data": {"base_uri": "https://airbyte.harvestapp.com", "full_domain": "airbyte.harvestapp.com", "name": "Airbyte", "is_active": true, "week_start_day": "Monday", "wants_timestamp_timers": false, "time_format": "hours_minutes", "date_format": "%m/%d/%Y", "plan_type": "simple-v4", "expense_feature": true, "invoice_feature": true, "estimate_feature": true, "team_feature": true, "weekly_capacity": 144000, "approval_feature": true, "clock": "12h", "currency": "USD", "currency_code_display": "iso_code_none", "currency_symbol_display": "symbol_before", "decimal_symbol": ".", "thousands_separator": ",", "color_scheme": "orange"}, "emitted_at": 1690884271497} {"stream": "invoices", "data": {"id": 28174545, "client_key": "489645d5b2becebe06f7a696a4d0db6a8a1c8ff1", "number": "2", "purchase_order": "", "amount": 22000.0, "due_amount": 21500.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": null, "discount_amount": 0.0, "subject": "Subj", "notes": "", "state": "draft", "period_start": null, "period_end": null, "issue_date": "2021-05-25", "due_date": "2021-05-25", "payment_term": "upon receipt", "sent_at": null, "paid_at": null, "closed_at": null, "recurring_invoice_id": null, "created_at": "2021-05-25T16:17:55Z", "updated_at": "2021-05-26T09:07:06Z", "paid_date": null, "currency": "USD", "payment_options": [], "client": {"id": 10748670, "name": "[SAMPLE] Client A"}, "estimate": null, "retainer": null, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": [{"id": 132632435, "kind": "Service", "description": "[SAMPLE] Fixed Fee Project", "quantity": 1.0, "unit_price": 21900.0, "amount": 21900.0, "taxed": false, "taxed2": false, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}}, {"id": 132632436, "kind": "Product", "description": "", "quantity": 1.0, "unit_price": 100.0, "amount": 100.0, "taxed": false, "taxed2": false, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}}]}, "emitted_at": 1690884271995} {"stream": "invoices", "data": {"id": 28174531, "client_key": "1a3a59c71a8dd22b3a341807456c754220dc202c", "number": "1", "purchase_order": "", "amount": 76.9, "due_amount": 0.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": 4.0, "discount_amount": 3.2, "subject": "", "notes": "Note", "state": "paid", "period_start": "2021-05-05", "period_end": "2021-05-05", "issue_date": "2021-05-25", "due_date": "2021-05-25", "payment_term": "upon receipt", "sent_at": "2021-05-25T16:46:28Z", "paid_at": "2021-05-25T00:00:00Z", "closed_at": null, "recurring_invoice_id": null, "created_at": "2021-05-25T16:16:51Z", "updated_at": "2021-05-26T09:06:37Z", "paid_date": "2021-05-25", "currency": "USD", "payment_options": [], "client": {"id": 10749825, "name": "First client"}, "estimate": null, "retainer": null, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": [{"id": 132632398, "kind": "Service", "description": "[FP] First project: Design (05/05/2021 - 05/05/2021)", "quantity": 0.01, "unit_price": 10.0, "amount": 0.1, "taxed": false, "taxed2": false, "project": {"id": 28674500, "name": "First project", "code": "FP"}}, {"id": 132632399, "kind": "Service", "description": "[FP] First project: Programming (05/05/2021 - 05/05/2021)", "quantity": 8.0, "unit_price": 10.0, "amount": 80.0, "taxed": false, "taxed2": false, "project": {"id": 28674500, "name": "First project", "code": "FP"}}]}, "emitted_at": 1690884271995} -{"stream": "invoice_messages", "data": {"id": 57176997, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "include_link_to_client_invoice": false, "send_me_a_copy": true, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:46:28Z", "updated_at": "2021-05-25T16:46:28Z", "attach_pdf": false, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1690884273321} -{"stream": "invoice_messages", "data": {"id": 57176927, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "include_link_to_client_invoice": false, "send_me_a_copy": true, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:43:30Z", "updated_at": "2021-05-25T16:43:30Z", "attach_pdf": true, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThe detailed invoice is attached as a PDF.\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1690884273322} +{"stream": "invoice_messages", "data": {"id": 57176997, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "send_me_a_copy": false, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:46:28Z", "updated_at": "2021-05-25T16:46:28Z", "attach_pdf": false, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "include_link_to_client_invoice": false, "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1708017738014} +{"stream": "invoice_messages", "data": {"id": 57176927, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "send_me_a_copy": false, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:43:30Z", "updated_at": "2021-05-25T16:43:30Z", "attach_pdf": true, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "include_link_to_client_invoice": false, "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThe detailed invoice is attached as a PDF.\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1708017738015} {"stream": "invoice_payments", "data": {"id": 21857618, "amount": 500.0, "paid_at": "2021-05-26T00:00:00Z", "recorded_by": "Airbyte Developer", "recorded_by_email": "integration-test@airbyte.io", "notes": "", "transaction_id": null, "created_at": "2021-05-26T09:07:06Z", "updated_at": "2021-05-26T09:07:06Z", "paid_date": "2021-05-26", "payment_gateway": {"id": null, "name": null}, "parent_id": 28174545}, "emitted_at": 1690884275279} {"stream": "invoice_payments", "data": {"id": 21857615, "amount": 76.9, "paid_at": "2021-05-25T00:00:00Z", "recorded_by": "Airbyte Developer", "recorded_by_email": "integration-test@airbyte.io", "notes": "Payed", "transaction_id": null, "created_at": "2021-05-26T09:06:37Z", "updated_at": "2021-05-26T09:06:37Z", "paid_date": "2021-05-25", "payment_gateway": {"id": null, "name": null}, "parent_id": 28174531}, "emitted_at": 1690884276439} {"stream": "invoice_item_categories", "data": {"id": 2732435, "name": "Product", "use_as_service": false, "use_as_expense": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1690884276919} diff --git a/airbyte-integrations/connectors/source-harvest/metadata.yaml b/airbyte-integrations/connectors/source-harvest/metadata.yaml index 58f00b3763c5a..842b3f385b13f 100644 --- a/airbyte-integrations/connectors/source-harvest/metadata.yaml +++ b/airbyte-integrations/connectors/source-harvest/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: fe2b4084-3386-4d3b-9ad6-308f61a6f1e6 - dockerImageTag: 0.1.22 + dockerImageTag: 0.1.23 dockerRepository: airbyte/source-harvest documentationUrl: https://docs.airbyte.com/integrations/sources/harvest githubIssueLabel: source-harvest @@ -24,7 +24,6 @@ data: registries: cloud: enabled: true - dockerImageTag: 0.1.21 oss: enabled: true releaseStage: generally_available diff --git a/airbyte-integrations/connectors/source-harvest/poetry.lock b/airbyte-integrations/connectors/source-harvest/poetry.lock index 14b727bd163fb..1ab272398503f 100644 --- a/airbyte-integrations/connectors/source-harvest/poetry.lock +++ b/airbyte-integrations/connectors/source-harvest/poetry.lock @@ -2,17 +2,17 @@ [[package]] name = "airbyte-cdk" -version = "0.55.2" +version = "0.62.1" description = "A framework for writing Airbyte Connectors." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte-cdk-0.55.2.tar.gz", hash = "sha256:6b0d4db12505bf724811111851577960d98ba28019bd29ad51912885b86cab42"}, - {file = "airbyte_cdk-0.55.2-py3-none-any.whl", hash = "sha256:af153a4ab3da54bb54ae5e7c1e6efa78ab5d76f038bd5ec3277948544b8b3212"}, + {file = "airbyte-cdk-0.62.1.tar.gz", hash = "sha256:3c934dd8b045079a9c807f699ca2012eaa5df755606e3f5b8b16247cbbd7e8c6"}, + {file = "airbyte_cdk-0.62.1-py3-none-any.whl", hash = "sha256:792399a602b7f5c3cd4ed2a5fce5910cfe3676b9b9199b9208f2d5236f5f42d3"}, ] [package.dependencies] -airbyte-protocol-models = "0.4.2" +airbyte-protocol-models = "0.5.1" backoff = "*" cachetools = "*" Deprecated = ">=1.2,<2.0" @@ -22,7 +22,7 @@ isodate = ">=0.6.1,<0.7.0" Jinja2 = ">=3.1.2,<3.2.0" jsonref = ">=0.2,<1.0" jsonschema = ">=3.2.0,<3.3.0" -pendulum = "*" +pendulum = "<3.0.0" pydantic = ">=1.10.8,<2.0.0" pyrate-limiter = ">=3.1.0,<3.2.0" python-dateutil = "*" @@ -39,13 +39,13 @@ vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embedding [[package]] name = "airbyte-protocol-models" -version = "0.4.2" +version = "0.5.1" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, - {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, ] [package.dependencies] @@ -478,102 +478,37 @@ files = [ [[package]] name = "pendulum" -version = "3.0.0" +version = "2.1.2" description = "Python datetimes made easy" optional = false -python-versions = ">=3.8" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ - {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, - {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, - {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, - {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, - {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, - {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, - {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, - {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, - {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, - {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, - {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, - {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, - {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, - {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, - {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, - {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, - {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, - {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, - {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, - {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, - {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, - {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, - {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, - {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, - {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, - {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, - {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, - {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, - {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, - {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, - {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, - {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, - {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, - {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, - {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, - {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, - {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, - {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, - {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, - {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, - {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, - {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, - {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, - {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, - {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, - {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, - {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, - {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, - {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, - {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, - {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, - {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, - {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, - {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, - {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, - {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, - {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, - {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, - {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, - {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, - {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, - {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, - {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, - {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, - {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, - {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, - {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, - {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, - {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, - {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, - {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, - {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, - {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, - {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, - {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, - {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, - {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, - {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, - {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, - {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, - {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, - {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, - {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, ] [package.dependencies] -python-dateutil = ">=2.6" -tzdata = ">=2020.1" - -[package.extras] -test = ["time-machine (>=2.6.0)"] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" [[package]] name = "platformdirs" @@ -779,6 +714,17 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + [[package]] name = "pyyaml" version = "6.0.1" @@ -958,17 +904,6 @@ files = [ {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] -[[package]] -name = "tzdata" -version = "2024.1" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -files = [ - {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, - {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, -] - [[package]] name = "url-normalize" version = "1.4.3" @@ -1096,4 +1031,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "ed3603077c0460053be9a7519c5a5fd3af246b2c310bb3714d3dcc4026ee0382" +content-hash = "c286c47fbc557061975b5d48e9b98b439952f4a2678139b5584efa12f9735165" diff --git a/airbyte-integrations/connectors/source-harvest/pyproject.toml b/airbyte-integrations/connectors/source-harvest/pyproject.toml index 1ef8e511b1168..e44d0c4887962 100644 --- a/airbyte-integrations/connectors/source-harvest/pyproject.toml +++ b/airbyte-integrations/connectors/source-harvest/pyproject.toml @@ -17,7 +17,7 @@ include = "source_harvest" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "==0.55.2" +airbyte-cdk = ">=0.62.1" [tool.poetry.scripts] source-harvest = "source_harvest.run:run" diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py index 3e7d1b2e56178..a910073440775 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py @@ -75,7 +75,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class IncrementalHarvestStream(HarvestStream, ABC): cursor_field = "updated_at" - def __init__(self, replication_start_date: pendulum.datetime = None, **kwargs): + def __init__(self, replication_start_date: Optional[pendulum.DateTime] = None, **kwargs) -> None: super().__init__(**kwargs) self._replication_start_date = replication_start_date @@ -96,7 +96,12 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - replication_start_date = stream_state.get(self.cursor_field) or self._replication_start_date + + replication_start_date = None + if stream_state.get(self.cursor_field): + replication_start_date = stream_state.get(self.cursor_field) + elif self._replication_start_date: + replication_start_date = self._replication_start_date.format("YYYY-MM-DDTHH:mm:ssZ") params.update({"updated_since": replication_start_date}) return params diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/integration/__init__.py b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/config.py new file mode 100644 index 0000000000000..a4f55f7898bf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/config.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime +from typing import Any, Dict + + +class ConfigBuilder: + def __init__(self) -> None: + self._config: Dict[str, Any] = { + "account_id": "an account id", + "replication_start_date": "2021-01-01T00:00:00Z", + "credentials": { + "api_token": "an api key" + } + } + + def with_account_id(self, account_id: str) -> "ConfigBuilder": + self._config["account_id"] = account_id + return self + + def with_replication_start_date(self, replication_start_date: datetime) -> "ConfigBuilder": + self._config["start_date"] = replication_start_date.isoformat()[:-13]+"Z" + return self + + def with_api_token(self, api_token: str) -> "ConfigBuilder": + self._config["credentials"]["api_token"] = api_token + return self + + def build(self) -> Dict[str, Any]: + return self._config diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoice_messages.py b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoice_messages.py new file mode 100644 index 0000000000000..093c80610432e --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoice_messages.py @@ -0,0 +1,103 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from source_harvest import SourceHarvest + +_A_REPLICATION_START_DATE = "2021-01-01T00:00:00+00:00" +_AN_ACCOUNT_ID = "1209384" +_AN_API_KEY = "harvestapikey" +_AN_INVOICE_ID = "an-invoice-id" +_STREAM_NAME = "invoice_messages" +_TEMPLATE_NAME = "invoice_messages" +_INVOICES_TEMPLATE_NAME = "invoices" +_RECORDS_PATH = FieldPath("invoice_messages") +_INVOICES_RECORDS_PATH = FieldPath("invoices") + + +def _a_message() -> RecordBuilder: + return create_record_builder( + find_template(_TEMPLATE_NAME, __file__), + _RECORDS_PATH, + ) + + +def _invoice_messages_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_TEMPLATE_NAME, __file__), + _RECORDS_PATH, + ) + + +def _an_invoice() -> RecordBuilder: + return create_record_builder( + find_template(_INVOICES_TEMPLATE_NAME, __file__), + _INVOICES_RECORDS_PATH, + record_id_path=FieldPath("id"), + ) + + +def _invoices_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_INVOICES_TEMPLATE_NAME, __file__), + _INVOICES_RECORDS_PATH, + ) + + +def _read( + config_builder: ConfigBuilder, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + return read( + SourceHarvest(), + config_builder.build(), + CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build(), + state, + expecting_exception + ) + + +class InvoicesTest(TestCase): + @HttpMocker() + def test_given_replication_start_date_when_read_then_request_is_created_properly(self, http_mocker: HttpMocker): + http_mocker.get( + HttpRequest( + url="https://api.harvestapp.com/v2/invoices", + query_params={ + "per_page": "50", + }, + ), + _invoices_response().with_record(_an_invoice().with_id(_AN_INVOICE_ID)).build() + ) + http_mocker.get( + HttpRequest( + url=f"https://api.harvestapp.com/v2/invoices/{_AN_INVOICE_ID}/messages", + query_params={ + "per_page": "50", + "updated_since": _A_REPLICATION_START_DATE, + }, + ), + _invoices_response().with_record(_a_message()).build() + ) + + _read(ConfigBuilder().with_account_id(_AN_ACCOUNT_ID).with_api_token(_AN_API_KEY).with_replication_start_date(datetime.fromisoformat(_A_REPLICATION_START_DATE))) + + # endpoint is called diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoices.py b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoices.py new file mode 100644 index 0000000000000..835490a917d02 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoices.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +from airbyte_cdk.sources.source import TState +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from source_harvest import SourceHarvest + +_A_REPLICATION_START_DATE = "2021-01-01T00:00:00+00:00" +_AN_ACCOUNT_ID = "1209384" +_AN_API_KEY = "harvestapikey" +_STREAM_NAME = "invoices" +_TEMPLATE_NAME = "invoices" +_RECORDS_PATH = FieldPath("invoices") + + +def _an_invoice() -> RecordBuilder: + return create_record_builder( + find_template(_TEMPLATE_NAME, __file__), + _RECORDS_PATH, + ) + + +def _invoices_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_TEMPLATE_NAME, __file__), + _RECORDS_PATH, + ) + + +def _read( + config_builder: ConfigBuilder, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + return read( + SourceHarvest(), + config_builder.build(), + CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build(), + state, + expecting_exception + ) + + +class InvoicesTest(TestCase): + @HttpMocker() + def test_given_replication_start_date_when_read_then_request_is_created_properly(self, http_mocker: HttpMocker): + http_mocker.get( + HttpRequest( + url="https://api.harvestapp.com/v2/invoices", + query_params={ + "per_page": "50", + "updated_since": _A_REPLICATION_START_DATE, + }, + headers={ + "Authorization": f"Bearer {_AN_API_KEY}", + "Harvest-Account-ID": _AN_ACCOUNT_ID, + } + ), + _invoices_response().build() + ) + + _read(ConfigBuilder().with_account_id(_AN_ACCOUNT_ID).with_api_token(_AN_API_KEY).with_replication_start_date(datetime.fromisoformat(_A_REPLICATION_START_DATE))) + + # endpoint is called diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/resource/http/response/invoice_messages.json b/airbyte-integrations/connectors/source-harvest/unit_tests/resource/http/response/invoice_messages.json new file mode 100644 index 0000000000000..fb4cedccb1779 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/resource/http/response/invoice_messages.json @@ -0,0 +1,68 @@ +{ + "invoice_messages": [ + { + "id": 27835209, + "sent_by": "Bob Powell", + "sent_by_email": "bobpowell@example.com", + "sent_from": "Bob Powell", + "sent_from_email": "bobpowell@example.com", + "include_link_to_client_invoice": false, + "send_me_a_copy": false, + "thank_you": false, + "reminder": false, + "send_reminder_on": null, + "created_at": "2017-08-23T22:15:06Z", + "updated_at": "2017-08-23T22:15:06Z", + "attach_pdf": true, + "event_type": null, + "recipients": [ + { + "name": "Richard Roe", + "email": "richardroe@example.com" + } + ], + "subject": "Past due invoice reminder: #1001 from API Examples", + "body": "Dear Customer,\r\n\r\nThis is a friendly reminder to let you know that Invoice 1001 is 144 days past due. If you have already sent the payment, please disregard this message. If not, we would appreciate your prompt attention to this matter.\r\n\r\nThank you for your business.\r\n\r\nCheers,\r\nAPI Examples" + }, + { + "id": 27835207, + "sent_by": "Bob Powell", + "sent_by_email": "bobpowell@example.com", + "sent_from": "Bob Powell", + "sent_from_email": "bobpowell@example.com", + "include_link_to_client_invoice": false, + "send_me_a_copy": true, + "thank_you": false, + "reminder": false, + "send_reminder_on": null, + "created_at": "2017-08-23T22:14:49Z", + "updated_at": "2017-08-23T22:14:49Z", + "attach_pdf": true, + "event_type": null, + "recipients": [ + { + "name": "Richard Roe", + "email": "richardroe@example.com" + }, + { + "name": "Bob Powell", + "email": "bobpowell@example.com" + } + ], + "subject": "Invoice #1001 from API Examples", + "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1001\r\nIssue Date: 04/01/2017\r\nClient: 123 Industries\r\nP.O. Number: \r\nAmount: €288.90\r\nDue: 04/01/2017 (upon receipt)\r\n\r\nThe detailed invoice is attached as a PDF.\r\n\r\nThank you!\r\n---------------------------------------------" + } + ], + "per_page": 2000, + "total_pages": 1, + "total_entries": 2, + "next_page": null, + "previous_page": null, + "page": 1, + "links": { + "first": "https://api.harvestapp.com/api/v2/invoices/13150403/messages?page=1&per_page=2000", + "next": null, + "previous": null, + "last": "https://api.harvestapp.com/v2/invoices/13150403/messages?page=1&per_page=2000" + } +} diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/resource/http/response/invoices.json b/airbyte-integrations/connectors/source-harvest/unit_tests/resource/http/response/invoices.json new file mode 100644 index 0000000000000..3be9cee64a0c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/resource/http/response/invoices.json @@ -0,0 +1,148 @@ +{ + "invoices": [ + { + "id": 13150403, + "client_key": "21312da13d457947a217da6775477afee8c2eba8", + "number": "1001", + "purchase_order": "", + "amount": 288.9, + "due_amount": 288.9, + "tax": 5, + "tax_amount": 13.5, + "tax2": 2, + "tax2_amount": 5.4, + "discount": 10, + "discount_amount": 30, + "subject": "Online Store - Phase 1", + "notes": "Some notes about the invoice.", + "state": "open", + "period_start": "2017-03-01", + "period_end": "2017-03-01", + "issue_date": "2017-04-01", + "due_date": "2017-04-01", + "payment_term": "upon receipt", + "sent_at": "2017-08-23T22:25:59Z", + "paid_at": null, + "paid_date": null, + "closed_at": null, + "recurring_invoice_id": null, + "created_at": "2017-06-27T16:27:16Z", + "updated_at": "2017-08-23T22:25:59Z", + "currency": "EUR", + "payment_options": ["credit_card"], + "client": { + "id": 5735776, + "name": "123 Industries" + }, + "estimate": null, + "retainer": null, + "creator": { + "id": 1782884, + "name": "Bob Powell" + }, + "line_items": [ + { + "id": 53341602, + "kind": "Service", + "description": "03/01/2017 - Project Management: [9:00am - 11:00am] Planning meetings", + "quantity": 2, + "unit_price": 100, + "amount": 200, + "taxed": true, + "taxed2": true, + "project": { + "id": 14308069, + "name": "Online Store - Phase 1", + "code": "OS1" + } + }, + { + "id": 53341603, + "kind": "Service", + "description": "03/01/2017 - Programming: [1:00pm - 2:00pm] Importing products", + "quantity": 1, + "unit_price": 100, + "amount": 100, + "taxed": true, + "taxed2": true, + "project": { + "id": 14308069, + "name": "Online Store - Phase 1", + "code": "OS1" + } + } + ] + }, + { + "id": 13150378, + "client_key": "9e97f4a65c5b83b1fc02f54e5a41c9dc7d458542", + "number": "1000", + "purchase_order": "1234", + "amount": 10700.0, + "due_amount": 0.0, + "tax": 5.0, + "tax_amount": 500.0, + "tax2": 2.0, + "tax2_amount": 200.0, + "discount": null, + "discount_amount": 0.0, + "subject": "Online Store - Phase 1", + "notes": "Some notes about the invoice.", + "state": "paid", + "period_start": null, + "period_end": null, + "issue_date": "2017-02-01", + "due_date": "2017-03-03", + "payment_term": "custom", + "sent_at": "2017-02-01T07:00:00Z", + "paid_at": "2017-02-21T00:00:00Z", + "paid_date": "2017-02-21", + "closed_at": null, + "recurring_invoice_id": null, + "created_at": "2017-06-27T16:24:30Z", + "updated_at": "2017-06-27T16:24:57Z", + "currency": "USD", + "client": { + "id": 5735776, + "name": "123 Industries" + }, + "estimate": { + "id": 1439814 + }, + "retainer": null, + "creator": { + "id": 1782884, + "name": "Bob Powell" + }, + "line_items": [ + { + "id": 53341450, + "kind": "Service", + "description": "50% of Phase 1 of the Online Store", + "quantity": 100.0, + "unit_price": 100.0, + "amount": 10000.0, + "taxed": true, + "taxed2": true, + "project": { + "id": 14308069, + "name": "Online Store - Phase 1", + "code": "OS1" + } + } + ] + } + ], + "per_page": 2000, + "total_pages": 1, + "total_entries": 2, + "next_page": null, + "previous_page": null, + "page": 1, + "links": { + "first": "https://api.harvestapp.com/v2/invoices?page=1&per_page=2000", + "next": null, + "previous": null, + "last": "https://api.harvestapp.com/v2/invoices?page=1&per_page=2000" + } +} diff --git a/docs/integrations/sources/harvest.md b/docs/integrations/sources/harvest.md index 27f8afd82df96..0da8d64456621 100644 --- a/docs/integrations/sources/harvest.md +++ b/docs/integrations/sources/harvest.md @@ -79,27 +79,28 @@ The connector is restricted by the [Harvest rate limits](https://help.getharvest ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------- | -| 0.1.22 | 2024-02-12 | [35154](https://github.com/airbytehq/airbyte/pull/35154) | Manage dependencies with Poetry. | -| 0.1.21 | 2023-11-30 | [33003](https://github.com/airbytehq/airbyte/pull/33003) | Update expected records | -| 0.1.20 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 0.1.19 | 2023-07-26 | [28755](https://github.com/airbytehq/airbyte/pull/28755) | Changed parameters for Time Reports to use 365 days as opposed to 1 year | -| 0.1.18 | 2023-05-29 | [26714](https://github.com/airbytehq/airbyte/pull/26714) | Remove `authSpecification` from spec in favour of `advancedAuth` | -| 0.1.17 | 2023-03-03 | [22983](https://github.com/airbytehq/airbyte/pull/22983) | Specified date formatting in specification | -| 0.1.16 | 2023-02-07 | [22417](https://github.com/airbytehq/airbyte/pull/22417) | Turn on default HttpAvailabilityStrategy | -| 0.1.15 | 2023-01-27 | [22008](https://github.com/airbytehq/airbyte/pull/22008) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.1.14 | 2023-01-09 | [21151](https://github.com/airbytehq/airbyte/pull/21151) | Skip 403 FORBIDDEN for all stream | -| 0.1.13 | 2022-12-22 | [20810](https://github.com/airbytehq/airbyte/pull/20810) | Skip 403 FORBIDDEN for `EstimateItemCategories` stream | -| 0.1.12 | 2022-12-16 | [20572](https://github.com/airbytehq/airbyte/pull/20572) | Introduce replication end date | -| 0.1.11 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | -| 0.1.10 | 2022-08-08 | [15221](https://github.com/airbytehq/airbyte/pull/15221) | Added `parent_id` for all streams which have parent stream | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------| :-------------------------------------------------------------------------------- | +| 0.1.23 | 2024-02-19 | [35305](https://github.com/airbytehq/airbyte/pull/35305) | Fix pendulum parsing error | +| 0.1.22 | 2024-02-12 | [35154](https://github.com/airbytehq/airbyte/pull/35154) | Manage dependencies with Poetry. | +| 0.1.21 | 2023-11-30 | [33003](https://github.com/airbytehq/airbyte/pull/33003) | Update expected records | +| 0.1.20 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.1.19 | 2023-07-26 | [28755](https://github.com/airbytehq/airbyte/pull/28755) | Changed parameters for Time Reports to use 365 days as opposed to 1 year | +| 0.1.18 | 2023-05-29 | [26714](https://github.com/airbytehq/airbyte/pull/26714) | Remove `authSpecification` from spec in favour of `advancedAuth` | +| 0.1.17 | 2023-03-03 | [22983](https://github.com/airbytehq/airbyte/pull/22983) | Specified date formatting in specification | +| 0.1.16 | 2023-02-07 | [22417](https://github.com/airbytehq/airbyte/pull/22417) | Turn on default HttpAvailabilityStrategy | +| 0.1.15 | 2023-01-27 | [22008](https://github.com/airbytehq/airbyte/pull/22008) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.1.14 | 2023-01-09 | [21151](https://github.com/airbytehq/airbyte/pull/21151) | Skip 403 FORBIDDEN for all stream | +| 0.1.13 | 2022-12-22 | [20810](https://github.com/airbytehq/airbyte/pull/20810) | Skip 403 FORBIDDEN for `EstimateItemCategories` stream | +| 0.1.12 | 2022-12-16 | [20572](https://github.com/airbytehq/airbyte/pull/20572) | Introduce replication end date | +| 0.1.11 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | +| 0.1.10 | 2022-08-08 | [15221](https://github.com/airbytehq/airbyte/pull/15221) | Added `parent_id` for all streams which have parent stream | | 0.1.9 | 2022-08-04 | [15312](https://github.com/airbytehq/airbyte/pull/15312) | Fix `started_time` and `ended_time` format schema error and updated report slicing | -| 0.1.8 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | -| 0.1.6 | 2021-11-14 | [7952](https://github.com/airbytehq/airbyte/pull/7952) | Implement OAuth 2.0 support | -| 0.1.5 | 2021-09-28 | [5747](https://github.com/airbytehq/airbyte/pull/5747) | Update schema date-time fields | -| 0.1.4 | 2021-06-22 | [5701](https://github.com/airbytehq/airbyte/pull/5071) | Harvest normalization failure: fixing the schemas | -| 0.1.3 | 2021-06-22 | [4274](https://github.com/airbytehq/airbyte/pull/4274) | Fix wrong data type on `statement_key` in `clients` stream | -| 0.1.2 | 2021-06-07 | [4222](https://github.com/airbytehq/airbyte/pull/4222) | Correct specification parameter name | -| 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.1.0 | 2021-06-07 | [3709](https://github.com/airbytehq/airbyte/pull/3709) | Release Harvest connector! | +| 0.1.8 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | +| 0.1.6 | 2021-11-14 | [7952](https://github.com/airbytehq/airbyte/pull/7952) | Implement OAuth 2.0 support | +| 0.1.5 | 2021-09-28 | [5747](https://github.com/airbytehq/airbyte/pull/5747) | Update schema date-time fields | +| 0.1.4 | 2021-06-22 | [5701](https://github.com/airbytehq/airbyte/pull/5071) | Harvest normalization failure: fixing the schemas | +| 0.1.3 | 2021-06-22 | [4274](https://github.com/airbytehq/airbyte/pull/4274) | Fix wrong data type on `statement_key` in `clients` stream | +| 0.1.2 | 2021-06-07 | [4222](https://github.com/airbytehq/airbyte/pull/4222) | Correct specification parameter name | +| 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.1.0 | 2021-06-07 | [3709](https://github.com/airbytehq/airbyte/pull/3709) | Release Harvest connector! |