From b479497c502f1bc8de1c304ffb295f9b2cc0f6e2 Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Thu, 8 Feb 2024 13:45:28 -0500 Subject: [PATCH] Source Linnworks: CDK update (#34717) --- .../connectors/source-linnworks/Dockerfile | 38 -------- .../connectors/source-linnworks/README.md | 92 +++++++++++++++++-- .../acceptance-test-config.yml | 46 ++++++---- .../integration_tests/abnormal_state.json | 20 ++-- .../connectors/source-linnworks/metadata.yaml | 23 ++--- .../connectors/source-linnworks/setup.py | 4 +- .../schemas/processed_order_details.json | 30 +++--- .../schemas/processed_orders.json | 2 +- .../source_linnworks/schemas/stock_items.json | 20 ++-- .../schemas/stock_location_details.json | 2 +- .../schemas/stock_locations.json | 29 +++++- .../source_linnworks/source.py | 7 +- .../source_linnworks/spec.json | 5 +- docs/integrations/sources/linnworks.md | 15 +-- 14 files changed, 210 insertions(+), 123 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-linnworks/Dockerfile diff --git a/airbyte-integrations/connectors/source-linnworks/Dockerfile b/airbyte-integrations/connectors/source-linnworks/Dockerfile deleted file mode 100644 index c8b96529f82a..000000000000 --- a/airbyte-integrations/connectors/source-linnworks/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_linnworks ./source_linnworks - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.5 -LABEL io.airbyte.name=airbyte/source-linnworks diff --git a/airbyte-integrations/connectors/source-linnworks/README.md b/airbyte-integrations/connectors/source-linnworks/README.md index b5395f00515c..d8557c27c0ce 100644 --- a/airbyte-integrations/connectors/source-linnworks/README.md +++ b/airbyte-integrations/connectors/source-linnworks/README.md @@ -6,23 +6,28 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development ### Prerequisites + **To iterate on this connector, make sure to complete this prerequisites section.** #### Minimum Python version required `= 3.7.0` #### Build & Activate Virtual Environment and install dependencies + From this connector directory, create a virtual environment: -``` + +```bash python -m venv .venv ``` This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your development environment of choice. To activate it from the terminal, run: -``` + +```bash source .venv/bin/activate pip install -r requirements.txt pip install '.[tests]' ``` + If you are in an IDE, follow your IDE's instructions to activate the virtualenv. Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is @@ -31,6 +36,7 @@ If this is mumbo jumbo to you, don't worry about it, just put your deps in `setu should work as you expect. #### Create credentials + **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/linnworks) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linnworks/spec.json` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. @@ -40,7 +46,8 @@ See `integration_tests/sample_config.json` for a sample config file. and place them into `secrets/config.json`. ### Locally running the connector -``` + +```bash python main.py spec python main.py check --config secrets/config.json python main.py discover --config secrets/config.json @@ -49,23 +56,82 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image +#### Use `airbyte-ci` to build your connector + +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** ```bash -airbyte-ci connectors --name=source-linnworks build +airbyte-ci connectors --name source-linnworks build +``` + +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-linnworks:dev`. + +##### Customizing our build process + +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: + +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +``` + +#### Build your own connector image + +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. + +```Dockerfile +FROM airbyte/source-linnworks:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -An image will be built with the tag `airbyte/source-linnworks:dev`. +Please use this as an example. This is not optimized. + +2. Build your image: -**Via `docker build`:** ```bash docker build -t airbyte/source-linnworks:dev . +# Running the spec command against your patched connector +docker run airbyte/source-linnworks:dev spec ``` #### Run + Then run any of the connector commands as follows: -``` + +```bash docker run --rm airbyte/source-linnworks:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linnworks:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linnworks:dev discover --config /secrets/config.json @@ -73,23 +139,30 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat ``` ## Testing + You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): + ```bash airbyte-ci connectors --name=source-linnworks test ``` ### Customizing acceptance Tests + Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management + All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: + * required for your connector to work need to go to `MAIN_REQUIREMENTS` list. * required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector + You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? + 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-linnworks test` 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). 3. Make sure the `metadata.yaml` content is up to date. @@ -97,4 +170,3 @@ You've checked out the repo, implemented a million dollar feature, and you're re 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - diff --git a/airbyte-integrations/connectors/source-linnworks/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linnworks/acceptance-test-config.yml index ce4b7ac5c2cf..c7343f0e5d84 100644 --- a/airbyte-integrations/connectors/source-linnworks/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-linnworks/acceptance-test-config.yml @@ -1,24 +1,38 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-linnworks:dev -tests: +acceptance_tests: spec: - - spec_path: "source_linnworks/spec.json" + tests: + - spec_path: "source_linnworks/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "processed_orders" + bypass_reason: "No seed data in our sandbox account" + - name: "processed_order_details" + bypass_reason: "No seed data in our sandbox account" + - name: "stock_items" + bypass_reason: "No seed data in our sandbox account" + # Removing incremental tests for now as we have no seed data in our sandbox account for the two streams that support it + # incremental: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-linnworks/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-linnworks/integration_tests/abnormal_state.json index fc503ae2fd7f..bb51cc3fa496 100644 --- a/airbyte-integrations/connectors/source-linnworks/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-linnworks/integration_tests/abnormal_state.json @@ -1,8 +1,16 @@ -{ - "processed_orders": { - "dProcessedOn": "2050-01-01T00:00:00+00:00" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "dProcessedOn": "2050-01-01T00:00:00+00:00" }, + "stream_descriptor": { "name": "processed_orders" } + } }, - "processed_order_details": { - "ProcessedDateTime": "2050-01-01T00:00:00+00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "ProcessedDateTime": "2050-01-01T00:00:00+00:00" }, + "stream_descriptor": { "name": "processed_order_details" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-linnworks/metadata.yaml b/airbyte-integrations/connectors/source-linnworks/metadata.yaml index a40652bfd7ff..b02aa7f7ab8f 100644 --- a/airbyte-integrations/connectors/source-linnworks/metadata.yaml +++ b/airbyte-integrations/connectors/source-linnworks/metadata.yaml @@ -1,29 +1,30 @@ data: + ab_internal: + ql: 100 + sl: 100 + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 dockerRepository: airbyte/source-linnworks + documentationUrl: https://docs.airbyte.com/integrations/sources/linnworks githubIssueLabel: source-linnworks icon: linnworks.svg license: MIT name: Linnworks - remoteRegistries: - pypi: - enabled: false - # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. - packageName: airbyte-source-linnworks registries: cloud: enabled: true oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/linnworks + remoteRegistries: + pypi: + enabled: false + packageName: airbyte-source-linnworks + supportLevel: community tags: - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-linnworks/setup.py b/airbyte-integrations/connectors/source-linnworks/setup.py index 54eed575b5d4..a2d048bbd31a 100644 --- a/airbyte-integrations/connectors/source-linnworks/setup.py +++ b/airbyte-integrations/connectors/source-linnworks/setup.py @@ -5,9 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] +MAIN_REQUIREMENTS = ["airbyte-cdk", "vcrpy"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json index f7c206789d2d..24d1f172926a 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json @@ -26,7 +26,7 @@ "GeneralInfo": { "type": "object", "description": "General information about order", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Status": { "type": "integer", @@ -73,7 +73,7 @@ "description": "Order identifiers. [Prime | Scheduled]", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "IdentifierId": { "type": "integer", @@ -141,7 +141,7 @@ "ScheduledDelivery": { "type": "object", "description": "Scheduled delivery dates. Take priority over despatch by date", - "additionalProperties": false, + "additionalProperties": true, "properties": { "From": { "type": "string", @@ -179,7 +179,7 @@ "ShippingInfo": { "type": "object", "description": "Order shipping information", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Vendor": { "type": ["null", "string"], @@ -238,7 +238,7 @@ "CustomerInfo": { "type": "object", "description": "Order Customer information (Name, email etc)", - "additionalProperties": false, + "additionalProperties": true, "properties": { "ChannelBuyerName": { "type": "string", @@ -247,7 +247,7 @@ "Address": { "type": "object", "description": "Customer address", - "additionalProperties": false, + "additionalProperties": true, "properties": { "EmailAddress": { "type": "string", @@ -305,7 +305,7 @@ "BillingAddress": { "type": "object", "description": "Customer billing address", - "additionalProperties": false, + "additionalProperties": true, "properties": { "EmailAddress": { "type": "string", @@ -365,7 +365,7 @@ "TotalsInfo": { "type": "object", "description": "Order totals information", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkOrderId": { "type": "string", @@ -426,7 +426,7 @@ "description": "Extended properties of an order", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "RowId": { "type": "string", @@ -459,7 +459,7 @@ "description": "List of order items", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "ItemId": { "type": "string", @@ -507,7 +507,7 @@ "OnPurchaseOrder": { "type": "object", "description": "Purchase order bound to this item", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkPurchaseItemId": { "type": "string", @@ -648,7 +648,7 @@ "description": "List of order item options", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkOptionId": { "type": "string", @@ -698,7 +698,7 @@ "description": "List of BinRacks used for OrderItem", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Quantity": { "type": "integer", @@ -747,7 +747,7 @@ "type": "array", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "BoxId": { "type": "integer", @@ -818,7 +818,7 @@ "description": "List of order notes", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "OrderNoteId": { "type": "string", diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_orders.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_orders.json index c2a7addf04b5..7f0c798b7ab1 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_orders.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_orders.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkOrderID": { "type": "string", diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_items.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_items.json index 115c4ccdc98e..544b8ab7312a 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_items.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_items.json @@ -1,14 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Suppliers": { "type": "array", "description": "Suppliers", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "IsDefault": { "type": "boolean", @@ -82,11 +82,11 @@ "description": "Stock Levels", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Location": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "description": "Location ID", "properties": { "StockLocationId": { @@ -200,7 +200,7 @@ "description": "List of item descriptions", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkRowId": { "type": "string", @@ -234,7 +234,7 @@ "description": "List of extended properties", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkRowId": { "type": "string", @@ -264,7 +264,7 @@ "description": "List item titles", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkRowId": { "type": "string", @@ -298,14 +298,14 @@ "description": "List of item prices", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Rules": { "type": "array", "description": "Pricing rule", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "pkRowId": { "type": ["null", "integer"], @@ -369,7 +369,7 @@ "description": "Image urls", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Source": { "type": "string", diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_location_details.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_location_details.json index 6eb3c4976ba0..4a146974d3ab 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_location_details.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_location_details.json @@ -1,6 +1,6 @@ { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "Address1": { "type": "string", diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_locations.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_locations.json index dfdde21d0c11..01671d3518d2 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_locations.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/stock_locations.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "StockLocationId": { "type": "string", @@ -30,6 +30,33 @@ "IsWarehouseManaged": { "type": ["null", "boolean"], "description": "If the location is warehouse managed." + }, + "Address1": { + "type": ["null", "string"] + }, + "Address2": { + "type": ["null", "string"] + }, + "City": { + "type": ["null", "string"] + }, + "County": { + "type": ["null", "string"] + }, + "Country": { + "type": ["null", "string"] + }, + "ZipCode": { + "type": ["null", "string"] + }, + "CountInOrderUntilAcknowledgement": { + "type": ["null", "boolean"] + }, + "FulfilmentCenterDeductStockWhenProcessed": { + "type": ["null", "boolean"] + }, + "IsNotTrackable": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py b/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py index 733ca9cb314b..7192e2ec5826 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py @@ -17,11 +17,11 @@ class LinnworksAuthenticator(Oauth2Authenticator): def __init__( self, - token_refresh_endpoint: str, application_id: str, application_secret: str, token: str, token_expiry_date: pendulum.datetime = None, + token_refresh_endpoint: str = "https://api.linnworks.net/api/Auth/AuthorizeByApplication", access_token_name: str = "Token", expires_in_name: str = "TTL", server_name: str = "Server", @@ -36,16 +36,19 @@ def __init__( access_token_name=access_token_name, expires_in_name=expires_in_name, ) - + self.access_token_name = access_token_name self.application_id = application_id self.application_secret = application_secret + self.expires_in_name = expires_in_name self.token = token self.server_name = server_name + self.token_refresh_endpoint = token_refresh_endpoint def get_auth_header(self) -> Mapping[str, Any]: return {"Authorization": self.get_access_token()} def get_access_token(self): + if self.token_has_expired(): t0 = pendulum.now() token, expires_in, server = self.refresh_access_token() diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/spec.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/spec.json index 8e99bdcbc648..4ed0f2ee5d80 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/spec.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/spec.json @@ -5,7 +5,7 @@ "title": "Linnworks Spec", "type": "object", "required": ["application_id", "application_secret", "token", "start_date"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "application_id": { "title": "Application ID.", @@ -20,7 +20,8 @@ }, "token": { "title": "API Token", - "type": "string" + "type": "string", + "airbyte_secret": true }, "start_date": { "title": "Start Date", diff --git a/docs/integrations/sources/linnworks.md b/docs/integrations/sources/linnworks.md index a791b39a88ff..47f9aa6257b3 100644 --- a/docs/integrations/sources/linnworks.md +++ b/docs/integrations/sources/linnworks.md @@ -69,10 +69,11 @@ Rate limits for the Linnworks API vary across endpoints. Use the [links in the * ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :----------------------------------------------------- | :-------------------------------------------------------------------------- | -| 0.1.5 | Unknown | Unknown | Bump Version | -| 0.1.4 | 2021-11-24 | [8226](https://github.com/airbytehq/airbyte/pull/8226) | Source Linnworks: improve streams ProcessedOrders and ProcessedOrderDetails | -| 0.1.3 | 2021-11-24 | [8169](https://github.com/airbytehq/airbyte/pull/8169) | Source Linnworks: refactor stream StockLocations | -| 0.1.2 | 2021-11-23 | [8177](https://github.com/airbytehq/airbyte/pull/8177) | Source Linnworks: add stream ProcessedOrderDetails | -| 0.1.0 | 2021-11-09 | [7588](https://github.com/airbytehq/airbyte/pull/7588) | New Source: Linnworks | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------- | +| 0.1.6 | 2024-01-31 | [34717](https://github.com/airbytehq/airbyte/pull/34717) | Update CDK and migrate to base image | +| 0.1.5 | 2022-11-20 | [19865](https://github.com/airbytehq/airbyte/pull/19865) | Bump Version | +| 0.1.4 | 2021-11-24 | [8226](https://github.com/airbytehq/airbyte/pull/8226) | Source Linnworks: improve streams ProcessedOrders and ProcessedOrderDetails | +| 0.1.3 | 2021-11-24 | [8169](https://github.com/airbytehq/airbyte/pull/8169) | Source Linnworks: refactor stream StockLocations | +| 0.1.2 | 2021-11-23 | [8177](https://github.com/airbytehq/airbyte/pull/8177) | Source Linnworks: add stream ProcessedOrderDetails | +| 0.1.0 | 2021-11-09 | [7588](https://github.com/airbytehq/airbyte/pull/7588) | New Source: Linnworks |