Improve Python Build. (#4540)

* Remove old pip build command. Deconflict Pydantic. * Run test on ec2 instance.
airbytehq · Jul 5, 2021 · 71637a2 · 71637a2
1 parent a7196d4
commit 71637a2
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 5 deletions.
diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml
@@ -14,8 +14,32 @@ on:
         required: false
 
 jobs:
-  integration_test:
-    runs-on: ubuntu-20.04
+  start-test-runner:
+    name: Start Build EC2 Runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Start EC2 Runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          ec2-image-id: ami-04bd6e81239f4f3fb
+          ec2-instance-type: c5.2xlarge
+          subnet-id: subnet-0469a9e68a379c1d3
+          security-group-id: sg-0793f3c9413f21970
+  integration-test:
+    needs: start-test-runner
+    runs-on: ${{ needs.start-test-runner.outputs.label }}
     steps:
       - name: Search for valid connector name format
         id: regex
@@ -154,3 +178,25 @@ jobs:
           comment-id: ${{ github.event.inputs.comment-id }}
           body: |
             > :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+  # In case of self-hosted EC2 errors, remove this block.
+  stop-test-runner:
+    name: Stop Build EC2 Runner
+    needs:
+      - start-test-runner # required to get output from the start-runner job
+      - integration-test # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          label: ${{ needs.start-test-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-test-runner.outputs.ec2-instance-id }}
diff --git a/airbyte-integrations/bases/airbyte-protocol/setup.py b/airbyte-integrations/bases/airbyte-protocol/setup.py
@@ -33,5 +33,5 @@
     url="https://github.com/airbytehq/airbyte",
     packages=setuptools.find_packages(),
     package_data={"": ["models/yaml/*.yaml"]},
-    install_requires=["PyYAML==5.4", "pydantic==1.6.2"],
+    install_requires=["PyYAML==5.4", "pydantic==1.6.*"],
 )
diff --git a/airbyte-integrations/bases/base-python/Dockerfile b/airbyte-integrations/bases/base-python/Dockerfile
@@ -4,7 +4,6 @@ COPY --from=airbyte/integration-base:dev /airbyte /airbyte
 WORKDIR /airbyte/base_python_code
 COPY base_python ./base_python
 COPY setup.py ./
-RUN pip install pip==20.2
 RUN pip install .
 
 ENV AIRBYTE_SPEC_CMD "base-python spec"

diff --git a/airbyte-integrations/bases/base-python/setup.py b/airbyte-integrations/bases/base-python/setup.py
@@ -35,7 +35,7 @@
     package_data={"": ["models/yaml/*.yaml"]},
     install_requires=[
         "PyYAML==5.4",
-        "pydantic==1.6.2",
+        "pydantic==1.6.*",
         "airbyte-protocol",
         "jsonschema==2.6.0",
         "requests",