Add CDK publishing setup. (#3156)

Allow us to publish CDK versions on PyPi via Github workflows. This lets us: - Version CDK moving forward - Publish via Github actions to both PyPi and test PyPi for test releases. Note, this will not work on this branch as Github only detects new workflows after they are checked into master, so I was forced to develop and test this from a fork. Browse this PR or this PR to see this command in action. Added publishing instructions to the Release Document.
airbytehq · May 3, 2021 · ced2438 · ced2438
1 parent 4cf69ac
commit ced2438
Show file tree

Hide file tree

Showing 12 changed files with 189 additions and 22 deletions.
diff --git a/.github/workflows/publish-cdk-command.yml b/.github/workflows/publish-cdk-command.yml
@@ -0,0 +1,55 @@
+name: Publish CDK
+on:
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        description: 'Indicates whether this is a dry-run or not. A dry-run publishes to Test PyPi. A proper run publishes to actual PyPi servers.'
+        required: false
+      comment-id:
+        description: 'The comment-id of the slash command. Used to update the comment with the status.'
+        required: false
+
+jobs:
+  publish-cdk:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Link comment to workflow run
+        if: github.event.inputs.comment-id
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+      # Make use of env vars to dynamically set the PyPi url. Since the default is set to publish to production PyPi, only set the url if during
+      # a dry-run publish to the Test PyPi servers.
+      - name: Publish to test PyPi if dry-run.
+        if: github.event.inputs.dry-run != 'false'
+        run: |
+          echo ${{ github.event.inputs.dry-run }}
+          echo "pypi_url=https://test.pypi.org/legacy/" >> $GITHUB_ENV
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: Publish Python Package
+        uses: mariamrf/py-package-publish-action@v1.1.0
+        with:
+          python_version: '3.7.9'
+          pip_version: '21.1'
+          subdir: 'airbyte-cdk/python/'
+        env:
+          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+          TWINE_REPOSITORY_URL: ${{ env.pypi_url }}
+      - name: Add Success Comment
+        if: github.event.inputs.comment-id && success()
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+      - name: Add Failure Comment
+        if: github.event.inputs.comment-id && !success()
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml
@@ -18,6 +18,7 @@ jobs:
           commands: |
             test
             publish
+            publish-cdk
           static-args: |
             ref=${{ steps.getref.outputs.ref }}
             comment-id=${{ github.event.comment.id }}

diff --git a/airbyte-cdk/python/LICENSE.txt b/airbyte-cdk/python/LICENSE.txt
@@ -0,0 +1,19 @@
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/airbyte-cdk/python/README.md b/airbyte-cdk/python/README.md
@@ -0,0 +1,43 @@
+# Airbyte Python Connector Development Framework (CDK)
+
+The Airbyte Python CDK is a framework for rapidly developing production-grade Airbyte connectors.
+The CDK currently offers helpers specific for creating Airbyte source connectors for:
+* HTTP APIs (REST APIs, GraphQL, etc..)
+* Singer Taps
+* Generic Python sources (anything not covered by the above)
+
+The CDK provides an improved developer experience by providing basic implementation structure and abstracting away low-level glue boilerplate.
+
+This document is a general introduction to the CDK. Readers should have basic familiarity with the [Airbyte Specification](https://docs.airbyte.io/architecture/airbyte-specification) before proceeding.
+
+## Getting started
+Generate an empty connector using the code generator. First clone the Airbyte repository then from the repository root run
+```
+cd airbyte-integrations/connector-templates/generator
+npm run generate
+```
+
+then follow the interactive prompt. Next, find all `TODO`s in the generated project directory -- they're accompanied by lots of comments explaining what you'll need to do in order to implement your connector. Upon completing all TODOs properly, you should have a functioning connector.
+
+Additionally, you can follow [this tutorial](docs/tutorials/http_api_source.md) for a complete walkthrough of creating an HTTP connector using the Airbyte CDK.
+
+### Concepts & Documentation
+See the [overview docs](docs/concepts/overview.md) for a tour through what the API offers.
+
+### Airbyte Specification
+Find the reference docs for the Airbyte Specification (the interface for how sources and destinations interact) [here](https://docs.airbyte.io/architecture/airbyte-specification).
+
+### Example Connectors
+
+**HTTP Connectors**:
+* [Exchangerates API](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py)
+* [Stripe](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py)
+* [Slack](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-slack/source_slack/source.py)
+
+**Singer connectors**:
+* [Salesforce](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/source.py)
+* [Github](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-github-singer/source_github_singer/source.py)
+
+**Simple Python connectors using the barebones `Source` abstraction**:
+* [Google Sheets](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py)
+* [Mailchimp](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py)
diff --git a/airbyte-cdk/python/pyproject.toml b/airbyte-cdk/python/pyproject.toml
@@ -0,0 +1,8 @@
+# Defines Python build system settings.
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel"
+]
+
+build-backend = "setuptools.build_meta"
diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py
@@ -21,24 +21,62 @@
 # SOFTWARE.
 
 
-import setuptools
+import pathlib
+from setuptools import setup, find_packages
 
-setuptools.setup(
+# The directory containing this file
+HERE = pathlib.Path(__file__).parent
+
+# The text of the README file
+README = (HERE / "README.md").read_text()
+
+setup(
     name="airbyte-cdk",
-    version="0.1.0",
-    description="The Airbyte Connector Development Kit",
+    version="0.0.1",
+    description="Contains machinery to make it easy to write an Airbyte Connector in Python.",
+    long_description=README,
+    long_description_content_type="text/markdown",
     author="Airbyte",
     author_email="contact@airbyte.io",
+    license="MIT",
     url="https://github.com/airbytehq/airbyte",
-    packages=setuptools.find_packages(),
-    package_data={"": ["models/yaml/*.yaml"]},
+    classifiers=[
+            # This information is used when browsing on PyPi.
+
+            # Dev Status
+            'Development Status :: 3 - Alpha',
+
+            # Project Audience
+            'Intended Audience :: Developers',
+            'Topic :: Scientific/Engineering',
+            'Topic :: Software Development :: Libraries :: Python Modules',
+
+            'License :: OSI Approved :: MIT License',
+
+            # Python Version Support
+            'Programming Language :: Python :: 3.7',
+            'Programming Language :: Python :: 3.8',
+            'Programming Language :: Python :: 3.9',
+            'Programming Language :: Python :: 3.10',
+    ],
+    keywords='airbyte connectors-development-kit cdk',
+    project_urls={
+            'Documentation': 'https://docs.airbyte.io/',
+            'Source': 'https://github.com/airbytehq/airbyte',
+            'Tracker': 'https://github.com/airbytehq/airbyte/issues',
+    },
+    packages=find_packages(exclude=("unit_tests",)),
     install_requires=[
-        "backoff",
-        "jsonschema==2.6.0",
-        "pendulum",
-        "pydantic==1.6.1",
-        "pytest",
-        "PyYAML==5.4",
-        "requests",
-    ]
+            "backoff",
+            "jsonschema==2.6.0",
+            "pendulum",
+            "pydantic==1.6.1",
+            "pytest",
+            "PyYAML==5.4",
+            "requests",
+    ],
+    python_requires='>=3.7.9',
+    entry_points={
+            "console_scripts": ["base-python=base_python.entrypoint:main"],
+    },
 )
diff --git a/airbyte-integrations/bases/base-python/README.md b/airbyte-integrations/bases/base-python/README.md
@@ -19,10 +19,10 @@ npm run generate
 
 then follow the interactive prompt. Next, find all `TODO`s in the generated project directory -- they're accompanied by lots of comments explaining what you'll need to do in order to implement your connector. Upon completing all TODOs properly, you should have a functioning connector. 
 
-Additionally, you can follow [this tutorial](docs/tutorials/http_api_source.md) for a complete walkthrough of creating an HTTP connector using the Airbyte CDK.
+Additionally, you can follow [this tutorial](https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/docs/tutorials/http_api_source.md) for a complete walkthrough of creating an HTTP connector using the Airbyte CDK.
 
 ### Concepts & Documentation
-See the [overview docs](docs/concepts/overview.md) for a tour through what the API offers.  
+See the [overview docs](https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/docs/concepts/overview.md) for a tour through what the API offers.
 
 ### Example Connectors
 

diff --git a/...ce-test/src/main/java/io/airbyte/integrations/standardtest/source/StandardSourceTest.java b/...ce-test/src/main/java/io/airbyte/integrations/standardtest/source/StandardSourceTest.java
@@ -112,6 +112,7 @@ public abstract class StandardSourceTest {
       "airbyte/source-braintree-singer",
       "airbyte/source-salesforce-singer",
       "airbyte/source-stripe-singer",
+      "airbyte/source-exchange-rates",
       "airbyte/source-stripe",
       "airbyte/source-github-singer",
       "airbyte/source-gitlab-singer",
@@ -332,7 +333,6 @@ public void testIncrementalSyncWithState() throws Exception {
         .filter(m -> m.getType() == Type.STATE)
         .map(AirbyteMessage::getState)
         .collect(Collectors.toList());
-
     assertFalse(recordMessages.isEmpty(), "Expected the first incremental sync to produce records");
     assertFalse(stateMessages.isEmpty(), "Expected incremental sync to produce STATE messages");
     // TODO validate exact records
@@ -444,7 +444,6 @@ private List<AirbyteMessage> runRead(ConfiguredAirbyteCatalog catalog, JsonNode
 
     final AirbyteSource source = new DefaultAirbyteSource(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), pbf));
     final List<AirbyteMessage> messages = new ArrayList<>();
-
     source.start(sourceConfig, jobRoot);
     while (!source.isFinished()) {
       source.attemptRead().ifPresent(messages::add);

diff --git a/airbyte-integrations/connectors/source-exchange-rates/build.gradle b/airbyte-integrations/connectors/source-exchange-rates/build.gradle
@@ -12,7 +12,7 @@ airbyteStandardSourceTestFile {
     // For more information on standard source tests, see https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/testing-connectors
 
     // All these input paths must live inside this connector's directory (or subdirectories)
-    specPath = "source_exchange_rate/spec.json"
+    specPath = "source_exchange_rates/spec.json"
 
     // configPath points to a config file which matches the spec.json supplied above. secrets/ is gitignored by default, so place your config file
     // there (in case it contains any credentials)

diff --git a/airbyte-integrations/connectors/source-exchange-rates/sample_files/state.json b/airbyte-integrations/connectors/source-exchange-rates/sample_files/state.json
@@ -0,0 +1,3 @@
+{
+ "exchange_rates": {"date": "2021-04-28"}
+}
diff --git a/airbyte-integrations/connectors/source-exchange-rates/setup.py b/airbyte-integrations/connectors/source-exchange-rates/setup.py
@@ -29,6 +29,6 @@
     author="Airbyte",
     author_email="contact@airbyte.io",
     packages=find_packages(),
-    package_data={"": ["*.json"]},
-    install_requires=["airbyte-protocol", "pendulum>=2,<3"],
+    package_data={"": ["*.json", "schemas/*.json"]},
+    install_requires=["pendulum>=2,<3"],
 )
diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py
@@ -36,6 +36,7 @@ class ExchangeRates(HttpStream):
     # HttpStream related fields
     url_base = "https://api.ratesapi.io/"
     cursor_field = date_field_name
+    primary_key = ""
 
     def __init__(self, base: str, start_date: DateTime):
         super().__init__()
@@ -80,7 +81,7 @@ def chunk_date_range(start_date: DateTime) -> Iterable[Mapping[str, any]]:
     now = pendulum.now()
     while start_date < now:
         day_of_week = start_date.day_of_week
-        if day_of_week != pendulum.SATURDAY & day_of_week != pendulum.SUNDAY:
+        if day_of_week != pendulum.SATURDAY and day_of_week != pendulum.SUNDAY:
             days.append({"date": start_date.to_date_string()})
         start_date = start_date.add(days=1)