feat: update to latest datahub 0.10.2+ (#13)

Modernize the entire project, using the latest version of poetry, newer
linters, and of course the latest datahub version.

Also, enable automated dependency management via dependabot

Signed-off-by: Lucas Roesler <roesler.lucas@gmail.com>

.github/dependabot.yaml

@@ -0,0 +1,16 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    commit-message:
+      prefix: "chore"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every week
+      interval: "weekly"
+    commit-message:
+      prefix: ci
+

.github/workflows/ci.yaml

@@ -6,46 +6,37 @@ on:
       - synchronize
       - opened
       - reopened
-
-# concurrency:
-#   group: ${{ github.workflow }}-${{ github.ref }}
-#   cancel-in-progress: true
-
+
 jobs:
   test:
     name: Test
     runs-on: ubuntu-latest
     env:
       default_branch: regs/heads/master
-      python_version: 3.8
-      poetry_version: 1.0.0
     steps:
-      # minimum setup
       - name: Check out repository code
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
+      - name: Install poetry
+        run: pipx install poetry
+
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: ${{ env.python_version }}
+          python-version-file: '.python-version'
+          cache: 'poetry'
 
-      - name: Setup Poetry
-        uses: Gr1N/setup-poetry@v7
-
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: ~/.cache/pypoetry
-          key: poetry-${{ runner.os }}-${{ env.python_version }}-${{ hashFiles('poetry.lock') }}
+          key: poetry-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
 
       - name: Poetry version
         run: poetry --version
 
       - name: Install dependencies
         run: poetry install
 
-      - name: Lint
-        run: make format lint
-
-      - name: Run unit tests
-        run: make test
+      - name: Test
+        run: poetry run pytest

.github/workflows/pr-titles.yaml

@@ -14,8 +14,27 @@ jobs:
   validate:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2.3.4
-      - uses: deepakputhraya/action-pr-title@master
+      - uses: actions/github-script@v6
+        env:
+          # Ensure pull request titles match the Conventional Commits specification
+          # https://www.conventionalcommits.org/en/v1.0.0/
+          regex: '^(feat|fix|chore|ci|refactor|test|docs)(\(.*\))?!?:'
         with:
-          # Ensure pull request titles match the Conventional Commits specification https://www.conventionalcommits.org/en/v1.0.0/
-          regex: '^(feat|fix|chore|ci|refactor|test)(\(.*\))?!?:'
+          script: |
+            if (context.eventName != "pull_request") {
+              core.setFailed("This action only works on pull_request events");
+              return;
+            }
+            core.info(`Checking pull request title with regex: ${process.env.regex}`);
+            const regex = RegExp(process.env.regex);
+            const {data: pullRequest} = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.payload.pull_request.number
+            });
+            const title = pullRequest.title;
+            core.info(`Pull Request title: "${title}"`);
+            if (!regex.test(title)) {
+                core.setFailed(`Pull Request title "${title}" failed to pass match regex - ${regex}`);
+                return;
+            }

.github/workflows/release-please.yaml

@@ -1,34 +1,31 @@
-name: Release Please
-
 on:
-  push:
-    branches:
-      - main
-
+    push:
+      branches:
+        - main
+
+name: release-please
 jobs:
-  release-please:
-    runs-on: ubuntu-latest
-    outputs:
-      release_created: ${{ steps.release.outputs.release_created }}
-      tag_name: ${{ steps.release.outputs.tag_name }}
-
-    steps:
-      - uses: GoogleCloudPlatform/release-please-action@v2
-        id: release
-        with:
-          # use the CI token to pretend not to be a action
-          token: ${{ secrets.CONTIAMO_CI_TOKEN }}
-          release-type: python
-          package-name: ""
-          bump-minor-pre-major: true
-          changelog-types: |
-            [
-              {"type":"feat","section":"Features","hidden":false},
-              {"type":"fix","section":"Bug Fixes","hidden":false},
-              {"type":"chore","section":"Miscellaneous","hidden":false},
-              {"type":"docs","section":"Miscellaneous","hidden":false},
-              {"type":"refactor","section":"Miscellaneous","hidden":false}
-            ]
-      - name: Debug release output
-        run: |
-          echo '${{ toJSON(steps.release.outputs) }}'
+    release-please:
+      runs-on: ubuntu-latest
+      steps:
+        - uses: GoogleCloudPlatform/release-please-action@v3.7.8
+          id: release
+          with:
+            # use the CI token to pretend not to be a action
+            token: ${{ secrets.CONTIAMO_CI_TOKEN }}
+            release-type: python
+            package-name: ""
+            bump-minor-pre-major: true
+            changelog-types: |
+              [
+                {"type":"feat","section":"Features","hidden":false},
+                {"type":"fix","section":"Bug Fixes","hidden":false},
+                {"type":"chore","section":"Miscellaneous","hidden":false},
+                {"type":"docs","section":"Miscellaneous","hidden":false},
+                {"type":"refactor","section":"Miscellaneous","hidden":false}
+              ]
+  
+        - name: Debug release output
+          run: |
+            echo '${{ toJSON(steps.release.outputs) }}'
+  

.github/workflows/release.yaml

@@ -18,24 +18,23 @@ jobs:
       registry: ghcr.io
       name: datahub-sap-hana
     steps:
-      # minimum python setup
       - name: Check out repository code
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-
+      - name: Install poetry
+        run: pipx install poetry
+
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: ${{ env.python_version }}
-
-      - name: Setup Poetry
-        uses: Gr1N/setup-poetry@v7
+          python-version-file: '.python-version'
+          cache: 'poetry'
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: ~/.cache/pypoetry
-          key: poetry-${{ runner.os }}-${{ env.python_version }}-${{ hashFiles('poetry.lock') }}
+          key: poetry-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
 
       - name: Poetry version
         run: poetry --version

.gitignore

@@ -1,5 +1,7 @@
 .venv
 .pytest_cache
+.task
+.ruff_cache
 __pycache__
 dist
 /hana_mces.json

.python-version

@@ -0,0 +1 @@
+3.9.16

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "cSpell.words": [
+        "datahub",
+        "Hana",
+        "sqlalchemy"
+    ]
+}

Dockerfile

@@ -1,8 +1,4 @@
-FROM linkedin/datahub-ingestion:${DATAHUB:-v0.8.22}
-
-COPY requirements.txt requirements.txt
-RUN pip install -r requirements.txt
-
+FROM acryldata/datahub-ingestion:${DATAHUB:-v0.10.3.1}
 
 COPY dist/ ./dist/
 RUN pip install --no-index --find-links=dist/ datahub_sap_hana

README.md

@@ -8,29 +8,39 @@ Pre-built Wheels can be downloaded from the [Releases page](https://github.com/c
 
 Otherwise, you must install from source.
 
-## Try it out
+### Requirements
+You need the following tools pre-installed
+* [Task](https://taskfile.dev/#/installation)
+* [Poetry](https://python-poetry.org/docs/#installation)
+* [Pyenv](https://github.com/pyenv/pyenv#installation)
+  * Make sure to have the [required build packages installed for your OS](https://github.com/pyenv/pyenv/wiki#suggested-build-environment)
+  * And the [shims for your shell](https://github.com/pyenv/pyenv#set-up-your-shell-environment-for-pyenv)
 
-1. You will need Python 3.9, [Poetry](https://python-poetry.org/docs/#installation), and [`pyenv`](https://github.com/pyenv/pyenv).
 
-   Once you have `pyenv` and Poetry installed, you should run
+## Try it out
+
+1. Clone the project
 
    ```sh
-   pyenv install 3.9.7
-   pyenv local 3.9.7
-   poetry config virtualenvs.in-project true
+   git clone git@github.com:contiamo/datahub-sap-hana.git
+   cd datahub-sap-hana
    ```
 
-2. Clone the project
+2. You will need Python 3.10 or higher
+
+   Once you have `pyenv` and Poetry installed, you should run
 
    ```sh
-   git clone git@github.com:contiamo/datahub-sap-hana.git
-   cd datahub-sap-hana
+   pyenv install 3.10.10
+   pyenv local 3.10.10
+   poetry config virtualenvs.in-project true
    ```
 
+
 3. Install the project and dependencies
 
    ```sh
-   make setup
+   task setup
    ```
 
 4. Edit the `examples/hana_recipe.yaml` to set the connection details to your SAP Hana database.
@@ -54,7 +64,7 @@ A Docker image with datahub and this package preinstalled is provided via the [G
 ```sh
 docker run -it --rm -v `pwd`:/opt \
    ghcr.io/contiamo/datahub-sap-hana:latest \
-   ingest run -c /opt/examples/ha.yaml
+   ingest run -c /opt/examples/hana_recipe.yaml
 ```
 
 Note that you may need to set the `--network` flag if you are using the Hana Express Docker image.
@@ -66,11 +76,11 @@ Note that you may need to set the `--network` flag if you are using the Hana Exp
 To run the unit tests, use
 
 ```sh
-make test
+task test
 ```
 
 To run all of the tests, just use
 
 ```sh
-poetry run pytest -v
+task test -- -v
 ```