diff --git a/airbyte-integrations/connectors/source-gutendex/.dockerignore b/airbyte-integrations/connectors/source-gutendex/.dockerignore deleted file mode 100644 index 55b5f4c1e88c..000000000000 --- a/airbyte-integrations/connectors/source-gutendex/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_gutendex -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-gutendex/README.md b/airbyte-integrations/connectors/source-gutendex/README.md index 3052e7a40b9a..93ce056ada29 100644 --- a/airbyte-integrations/connectors/source-gutendex/README.md +++ b/airbyte-integrations/connectors/source-gutendex/README.md @@ -1,7 +1,7 @@ # Gutendex source connector -This is the repository for the Gutendex source connector, written in Python. +This is the repository for the Gutendex configuration based source connector. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/gutendex). ## Local development diff --git a/airbyte-integrations/connectors/source-gutendex/integration_tests/__init__.py b/airbyte-integrations/connectors/source-gutendex/integration_tests/__init__.py index c941b3045795..66f6de8cb2bb 100644 --- a/airbyte-integrations/connectors/source-gutendex/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-gutendex/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-gutendex/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gutendex/integration_tests/acceptance.py index 82823254d266..efc25f08ce82 100644 --- a/airbyte-integrations/connectors/source-gutendex/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gutendex/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-gutendex/main.py b/airbyte-integrations/connectors/source-gutendex/main.py index 8304e254daa4..a3835fb6c758 100644 --- a/airbyte-integrations/connectors/source-gutendex/main.py +++ b/airbyte-integrations/connectors/source-gutendex/main.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. # from source_gutendex.run import run diff --git a/airbyte-integrations/connectors/source-gutendex/metadata.yaml b/airbyte-integrations/connectors/source-gutendex/metadata.yaml index e2d0b94856d5..84346fef9a80 100644 --- a/airbyte-integrations/connectors/source-gutendex/metadata.yaml +++ b/airbyte-integrations/connectors/source-gutendex/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: bff9a277-e01d-420d-81ee-80f28a307318 - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 dockerRepository: airbyte/source-gutendex githubIssueLabel: source-gutendex license: MIT diff --git a/airbyte-integrations/connectors/source-gutendex/pyproject.toml b/airbyte-integrations/connectors/source-gutendex/pyproject.toml index 4baad4d0e3c7..4f6bef513f1f 100644 --- a/airbyte-integrations/connectors/source-gutendex/pyproject.toml +++ b/airbyte-integrations/connectors/source-gutendex/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "0.1.2" +version = "0.1.3" name = "source-gutendex" description = "Source implementation for Gutendex." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-gutendex/requirements.txt b/airbyte-integrations/connectors/source-gutendex/requirements.txt deleted file mode 100644 index d6e1198b1ab1..000000000000 --- a/airbyte-integrations/connectors/source-gutendex/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/source-gutendex/source_gutendex/__init__.py b/airbyte-integrations/connectors/source-gutendex/source_gutendex/__init__.py index 386493aed04c..c5cad6d07af6 100644 --- a/airbyte-integrations/connectors/source-gutendex/source_gutendex/__init__.py +++ b/airbyte-integrations/connectors/source-gutendex/source_gutendex/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-gutendex/source_gutendex/manifest.yaml b/airbyte-integrations/connectors/source-gutendex/source_gutendex/manifest.yaml index 8b7c03421a25..bd46f0590e03 100644 --- a/airbyte-integrations/connectors/source-gutendex/source_gutendex/manifest.yaml +++ b/airbyte-integrations/connectors/source-gutendex/source_gutendex/manifest.yaml @@ -1,50 +1,236 @@ -version: "0.29.0" +version: 0.78.1 + +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - books definitions: - selector: - extractor: - field_path: - - results - requester: - url_base: "https://gutendex.com/" - http_method: "GET" - request_parameters: - author_year_start: "{{ config['author_year_start'] }}" - author_year_end: "{{ config['author_year_end'] }}" - copyright: "{{ config['copyright'] }}" - languages: "{{ config['languages'] }}" - search: "{{ config['search'] }}" - sort: "{{ config['sort'] }}" - topic: "{{ config['topic'] }}" - retriever: - record_selector: - $ref: "#/definitions/selector" - paginator: - type: "DefaultPaginator" - pagination_strategy: - type: "PageIncrement" - page_size: 32 - page_token_option: - type: RequestOption - inject_into: "request_parameter" - field_name: "page" - page_size_option: - inject_into: "body_data" - field_name: "page_size" - requester: - $ref: "#/definitions/requester" - base_stream: - retriever: - $ref: "#/definitions/retriever" - books_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "books" - path: "/books" + streams: + books: + type: DeclarativeStream + name: books + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: /books + http_method: GET + request_parameters: + author_year_start: "{{ config['author_year_start'] }}" + author_year_end: "{{ config['author_year_end'] }}" + copyright: "{{ config['copyright'] }}" + languages: "{{ config['languages'] }}" + search: "{{ config['search'] }}" + sort: "{{ config['sort'] }}" + topic: "{{ config['topic'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + inject_into: body_data + field_name: page_size + pagination_strategy: + type: PageIncrement + page_size: 32 + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/books" + base_requester: + type: HttpRequester + url_base: https://gutendex.com/ streams: - - "#/definitions/books_stream" + - $ref: "#/definitions/streams/books" -check: - stream_names: - - "books" +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: [] + properties: + author_year_end: + type: string + description: >- + (Optional) Defines the maximum birth year of the authors. Books by + authors born after the end year will not be returned. Supports both + positive (CE) or negative (BCE) integer values + pattern: ^[-]?[0-9]{1,4}$ + examples: + - 2002 + - 500 + - -500 + - 2020 + order: 0 + author_year_start: + type: string + description: >- + (Optional) Defines the minimum birth year of the authors. Books by + authors born prior to the start year will not be returned. Supports + both positive (CE) or negative (BCE) integer values + pattern: ^[-]?[0-9]{1,4}$ + examples: + - 2002 + - 500 + - -500 + - 2020 + order: 1 + copyright: + type: string + description: >- + (Optional) Use this to find books with a certain copyright status - + true for books with existing copyrights, false for books in the public + domain in the USA, or null for books with no available copyright + information. + pattern: ^(true|false|null)$ + examples: + - true + - false + - null + order: 2 + languages: + type: string + description: >- + (Optional) Use this to find books in any of a list of languages. They + must be comma-separated, two-character language codes. + examples: + - en + - en,fr,fi + order: 3 + search: + type: string + description: >- + (Optional) Use this to search author names and book titles with given + words. They must be separated by a space (i.e. %20 in URL-encoded + format) and are case-insensitive. + examples: + - dickens%20great%20expect + - dickens + order: 4 + sort: + type: string + description: >- + (Optional) Use this to sort books - ascending for Project Gutenberg ID + numbers from lowest to highest, descending for IDs highest to lowest, + or popular (the default) for most popular to least popular by number + of downloads. + pattern: ^(ascending|descending|popular)$ + examples: + - ascending + - descending + - popular + order: 5 + topic: + type: string + description: >- + (Optional) Use this to search for a case-insensitive key-phrase in + books' bookshelves or subjects. + examples: + - children + - fantasy + order: 6 + additionalProperties: true + +metadata: + autoImportSchema: + books: false + +schemas: + books: + type: object + $schema: http://json-schema.org/draft-04/schema# + additionalProperties: true + properties: + authors: + type: array + items: + type: object + properties: + birth_year: + type: + - integer + - "null" + death_year: + type: + - integer + - "null" + name: + type: string + required: + - name + - birth_year + - death_year + bookshelves: + type: array + items: + type: string + copyright: + type: + - boolean + - "null" + download_count: + type: integer + formats: + type: object + properties: + application/epub+zip: + type: string + application/octet-stream: + type: string + application/rdf+xml: + type: string + application/x-mobipocket-ebook: + type: string + image/jpeg: + type: string + text/html: + type: string + text/plain; charset=us-ascii: + type: string + id: + type: integer + languages: + type: array + items: + type: string + media_type: + type: string + subjects: + type: array + items: + type: string + title: + type: string + translators: + type: array + items: + type: object + properties: + birth_year: + type: + - integer + - "null" + death_year: + type: + - integer + - "null" + name: + type: string + required: + - name + - birth_year + - death_year diff --git a/airbyte-integrations/connectors/source-gutendex/source_gutendex/run.py b/airbyte-integrations/connectors/source-gutendex/source_gutendex/run.py index ba4bcb6755ac..2d5c673efb13 100644 --- a/airbyte-integrations/connectors/source-gutendex/source_gutendex/run.py +++ b/airbyte-integrations/connectors/source-gutendex/source_gutendex/run.py @@ -1,12 +1,13 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. # import sys from airbyte_cdk.entrypoint import launch -from source_gutendex import SourceGutendex + +from .source import SourceGutendex def run(): diff --git a/airbyte-integrations/connectors/source-gutendex/source_gutendex/schemas/TODO.md b/airbyte-integrations/connectors/source-gutendex/source_gutendex/schemas/TODO.md deleted file mode 100644 index b040faf128f4..000000000000 --- a/airbyte-integrations/connectors/source-gutendex/source_gutendex/schemas/TODO.md +++ /dev/null @@ -1,19 +0,0 @@ -# TODO: Define your stream schemas - -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -You can describe the schema of your streams using one `.json` file per stream. - -## Static schemas - -From the `gutendex.yaml` configuration file, you read the `.json` files in the `schemas/` directory. You can refer to a schema in your configuration file using the `schema_loader` component's `file_path` field. For example: - -``` -schema_loader: - type: JsonSchema - file_path: "./source_gutendex/schemas/customers.json" -``` - -Every stream specified in the configuration file should have a corresponding `.json` schema file. - -Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-gutendex/source_gutendex/schemas/books.json b/airbyte-integrations/connectors/source-gutendex/source_gutendex/schemas/books.json deleted file mode 100644 index 715c849065bd..000000000000 --- a/airbyte-integrations/connectors/source-gutendex/source_gutendex/schemas/books.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "title": { - "type": "string" - }, - "authors": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "birth_year": { - "type": ["integer", "null"] - }, - "death_year": { - "type": ["integer", "null"] - } - }, - "required": ["name", "birth_year", "death_year"] - } - }, - "translators": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "birth_year": { - "type": ["integer", "null"] - }, - "death_year": { - "type": ["integer", "null"] - } - }, - "required": ["name", "birth_year", "death_year"] - } - }, - "subjects": { - "type": "array", - "items": { - "type": "string" - } - }, - "bookshelves": { - "type": "array", - "items": { - "type": "string" - } - }, - "languages": { - "type": "array", - "items": { - "type": "string" - } - }, - "copyright": { - "type": ["boolean", "null"] - }, - "media_type": { - "type": "string" - }, - "formats": { - "type": "object", - "properties": { - "image/jpeg": { - "type": "string" - }, - "application/x-mobipocket-ebook": { - "type": "string" - }, - "application/rdf+xml": { - "type": "string" - }, - "text/html": { - "type": "string" - }, - "application/epub+zip": { - "type": "string" - }, - "text/plain; charset=us-ascii": { - "type": "string" - }, - "application/octet-stream": { - "type": "string" - } - } - }, - "download_count": { - "type": "integer" - } - }, - "required": [ - "id", - "title", - "authors", - "translators", - "subjects", - "bookshelves", - "languages", - "copyright", - "media_type", - "formats", - "download_count" - ] -} diff --git a/airbyte-integrations/connectors/source-gutendex/source_gutendex/source.py b/airbyte-integrations/connectors/source-gutendex/source_gutendex/source.py index e076b49b633b..98f94cdcc14e 100644 --- a/airbyte-integrations/connectors/source-gutendex/source_gutendex/source.py +++ b/airbyte-integrations/connectors/source-gutendex/source_gutendex/source.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. # from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource diff --git a/airbyte-integrations/connectors/source-gutendex/source_gutendex/spec.yaml b/airbyte-integrations/connectors/source-gutendex/source_gutendex/spec.yaml deleted file mode 100644 index f22b1645b25e..000000000000 --- a/airbyte-integrations/connectors/source-gutendex/source_gutendex/spec.yaml +++ /dev/null @@ -1,59 +0,0 @@ -documentationUrl: https://docs.airbyte.com/integrations/sources/gutendex -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Gutendex Spec - type: object - additionalProperties: true - properties: - author_year_start: - type: string - description: (Optional) Defines the minimum birth year of the authors. Books by authors born prior to the start year will not be returned. Supports both positive (CE) or negative (BCE) integer values - pattern: ^[-]?[0-9]{1,4}$ - examples: - - 2002 - - 500 - - -500 - - 2020 - author_year_end: - type: string - description: (Optional) Defines the maximum birth year of the authors. Books by authors born after the end year will not be returned. Supports both positive (CE) or negative (BCE) integer values - pattern: ^[-]?[0-9]{1,4}$ - examples: - - 2002 - - 500 - - -500 - - 2020 - copyright: - type: string - description: (Optional) Use this to find books with a certain copyright status - true for books with existing copyrights, false for books in the public domain in the USA, or null for books with no available copyright information. - pattern: ^(true|false|null)$ - examples: - - true - - false - - null - languages: - type: string - description: (Optional) Use this to find books in any of a list of languages. They must be comma-separated, two-character language codes. - examples: - - en - - en,fr,fi - search: - type: string - description: (Optional) Use this to search author names and book titles with given words. They must be separated by a space (i.e. %20 in URL-encoded format) and are case-insensitive. - examples: - - dickens%20great%20expect - - dickens - sort: - type: string - description: (Optional) Use this to sort books - ascending for Project Gutenberg ID numbers from lowest to highest, descending for IDs highest to lowest, or popular (the default) for most popular to least popular by number of downloads. - pattern: ^(ascending|descending|popular)$ - examples: - - ascending - - descending - - popular - topic: - type: string - description: (Optional) Use this to search for a case-insensitive key-phrase in books' bookshelves or subjects. - examples: - - children - - fantasy diff --git a/docs/integrations/sources/gutendex.md b/docs/integrations/sources/gutendex.md index 3e0eb2b42e24..218f7c41e51e 100644 --- a/docs/integrations/sources/gutendex.md +++ b/docs/integrations/sources/gutendex.md @@ -104,10 +104,11 @@ There is no published rate limit. However, since this data updates infrequently,
Expand to review -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :-------------------------------------------------------- | :----------------------------------------- | -| 0.1.2 | 2024-06-04 | [39017](https://github.com/airbytehq/airbyte/pull/39017) | [autopull] Upgrade base image to v1.2.1 | +| Version | Date | Pull Request | Subject | +| :------ |:-----------| :-------------------------------------------------------- |:--------------------------------------------| +| 0.1.3 | 2024-06-15 | [39509](https://github.com/airbytehq/airbyte/pull/39509) | Make connector compatible with Builder | +| 0.1.2 | 2024-06-04 | [39017](https://github.com/airbytehq/airbyte/pull/39017) | [autopull] Upgrade base image to v1.2.1 | | 0.1.1 | 2024-05-21 | [38509](https://github.com/airbytehq/airbyte/pull/38509) | [autopull] base image + poetry + up_to_date | -| 0.1.0 | 2022-10-17 | [#18075](https://github.com/airbytehq/airbyte/pull/18075) | 🎉 New Source: Gutendex API [low-code CDK] | +| 0.1.0 | 2022-10-17 | [#18075](https://github.com/airbytehq/airbyte/pull/18075) | 🎉 New Source: Gutendex API [low-code CDK] |
\ No newline at end of file