Feature/dry ci build

.github/workflows/build-golang-macos.yaml

@@ -1,10 +1,7 @@
 ---
-name: macOS -- Build Python wheel using golang's gopy
+name: macOS build
 
-on:
-  push:
-      branches:
-          - main
+on: [push]
 
 defaults:
   run:
@@ -17,102 +14,37 @@ jobs:
   build:
     runs-on: macos-13
     permissions:
-        contents: write
+      contents: write
 
     strategy:
-        fail-fast: true
-        matrix:
-            go-version: [1.22.x]
+      fail-fast: true
+      matrix:
+        go-version: [1.22.x]
+        python3_version: [ "3.11", "3.12" ]
 
     steps:
-        - uses: actions/checkout@v4
-
-        # - name: Setup Docker on macOS using Colima, Lima-VM, and Homebrew
-        #   uses: douglascamata/setup-docker-macos-action@v1-alpha
-
-
-        # - name: Verify docker Installation
-        #   run: |
-        #     docker --version
-        #     docker ps
-
-        # - name: Setup vagrant
-        #   run: |
-        #       brew install vagrant
-        #       vagrant --version
-
-        - name: Setup Go
-          uses: actions/setup-go@v4
-          with:
-              go-version: ${{ matrix.go-version }}
-              cache-dependency-path: go.sum
-        - name: Install dependencies
-          run: go get .
-        - name: Test with Go
-          run: go test -timeout 40s -run ^TestHello$ gotdf_python -count=1 # go test
-
-        - name: Set up Python
-          uses: actions/setup-python@v4
-          with:
-              python-version: '3.12'
-
-        - name: Install dependencies
-          run: |
-              pip install poetry
-
-              # Look for go/bin (skip, we know it exists)
-              echo '$HOME/'
-              ls -la "$HOME/"
-
-              echo '$HOME/.local/'
-              ls -la "$HOME/.local/"
-
-              echo '$HOME/go/'
-              ls -la "$HOME/go/"
-
-        - name: Add Go bin directory to PATH
-          run: echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV
-
-        # FIXME: Add more caching
-        - name: Add gopy dependencies and build wheel
-          run: |
-            ./ci-build.sh
-
-        - name: Test Python wheel
-          run: |
-              # Test wheel installation
-              pip install dist/otdf_python-0.1.10-py3-none-any.whl
-
-              # Test wheel functionality
-              # python3 validate_otdf_python.py
-
-        # - uses: ./.github/workflows/platform-integration-test.yaml
-        #   with:
-        #     wheel: dist/otdf_python-0.1.10-py3-none-any.whl
-
-  # release:
-  #   needs: build
-  #   runs-on: macos-latest
-  #   if: github.ref == 'refs/heads/main'
-  #   permissions:
-  #       contents: write
-  #   steps:
-  #       - uses: actions/checkout@v4
-
-        - name: Store version
-          run: |
-              pip install poetry
-
-              PROJECT_VESION=$(poetry version -s)
-              echo "PROJECT_VESION=$PROJECT_VESION" >> $GITHUB_ENV
-
-        - name: Install twine
-          run: pip install twine
-
-        - name: Upload to PyPI
-          env:
-            TWINE_UPLOAD_URL: https://upload.pypi.org/legacy/
-            PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
-            PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-          run: |
-            twine upload --password "$PYPI_PASSWORD" --user "$PYPI_USERNAME" --repository-url "$TWINE_UPLOAD_URL" dist/*
+      - uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go-version }}
+          cache-dependency-path: go.sum
+      - name: Install dependencies
+        run: go get .
+      - name: Test with Go
+        run: go test -timeout 40s -run ^TestHello$ gotdf_python -count=1 # go test
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python3_version }}
+
+      # FIXME: Add more caching
+      - name: Configure gopy / dependencies, and build wheel
+        run: |
+          ./ci-build.sh
+
+      # - uses: ./.github/workflows/platform-integration-test.yaml
+      #   with:
+      #     wheel: dist/otdf_python-0.1.11-py3-none-any.whl

.github/workflows/build-golang-ubuntu.yaml

@@ -1,92 +1,65 @@
 ---
-name: Ubuntu -- Build Python wheel using golang's gopy
+name: Ubuntu build
 
 on: [push]
 
 jobs:
-    call-lint:
-      uses: ./.github/workflows/lint-on-ubuntu.yaml
-
-    build:
-        runs-on: ubuntu-latest
-        permissions:
-            contents: write
-
-        strategy:
-            fail-fast: true
-            matrix:
-                go-version: [1.22.x]
-                # go-version: [ '1.19', '1.20', '1.22.x' ]
-
-        steps:
-            - uses: actions/checkout@v4
-
-
-            - name: Setup Go
-              uses: actions/setup-go@v4
-              with:
-                  go-version: ${{ matrix.go-version }}
-                  cache-dependency-path: go.sum
-            - name: Install dependencies
-              run: go get .
-            - name: Test with Go
-              run: go test -timeout 40s -run ^TestHello$ gotdf_python -count=1 # go test
-
-            - name: Set up Python
-              uses: actions/setup-python@v4
-              with:
-                  python-version: '3.12'
-
-            - name: Install dependencies
-              run: |
-                  pip install poetry
-
-                  # Look for go/bin (skip, we know it exists)
-                  echo '$HOME/'
-                  ls -la "$HOME/"
-
-                  echo '$HOME/.local/'
-                  ls -la "$HOME/.local/"
-
-                  echo '$HOME/go/'
-                  ls -la "$HOME/go/"
-
-            - name: Add Go bin directory to PATH
-              run: echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV
-
-            # FIXME: Add more caching
-            - name: Add gopy dependencies and build wheel
-              run: |
-                ./ci-build.sh
-
-            - name: Check existing workspace content
-              run: |
-                  ls -la
-                  ls -la dist/
-
-            - name: Test Python wheel
-              run: |
-                  # Test wheel installation
-                  pip install dist/otdf_python-0.1.10-py3-none-any.whl
-
-                  # DISABLED: Need to figure out Ubuntu nested VM
-                  # Test wheel functionality
-                  # python3 validate_otdf_python.py
-
-            - uses: actions/cache/restore@v4
-              with:
-                path: dist/otdf_python-0.1.10-py3-none-any.whl
-                key: ${{ runner.os }}-data-${{ github.sha }}
-
-            - uses: actions/cache/save@v4
-              with:
-                path: dist/otdf_python-0.1.10-py3-none-any.whl
-                key: ${{ runner.os }}-data-${{ github.sha }}
-                restore-keys: |
-                  ${{ runner.os }}-data-
-
-    integration-test:
-      needs: build
-      uses: ./.github/workflows/platform-integration-test.yaml
-      with:
-        wheel: dist/otdf_python-0.1.10-py3-none-any.whl
+  call-lint:
+    uses: ./.github/workflows/lint-on-ubuntu.yaml
+
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    strategy:
+      fail-fast: true
+      matrix:
+        go-version: [1.22.x]
+        python3_version: [ "3.11", "3.12" ]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go-version }}
+          cache-dependency-path: go.sum
+      - name: Install dependencies
+        run: go get .
+      - name: Test with Go
+        run: go test -timeout 40s -run ^TestHello$ gotdf_python -count=1 # go test
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python3_version }}
+
+      # FIXME: Add more caching
+      - name: Configure gopy / dependencies, and build wheel
+        run: |
+          ./ci-build.sh
+
+      - uses: actions/cache/restore@v4
+        with:
+          path: dist/otdf_python-0.1.11-py3-none-any.whl
+          key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }}
+
+      - uses: actions/cache/save@v4
+        with:
+          path: dist/otdf_python-0.1.11-py3-none-any.whl
+          key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}${{ matrix.python3_version }}-data-
+
+  integration-test:
+    strategy:
+      fail-fast: true
+      matrix:
+        python3_version: [ "3.11", "3.12" ]
+    needs: build
+    uses: ./.github/workflows/platform-integration-test.yaml
+    with:
+      wheel: dist/otdf_python-0.1.11-py3-none-any.whl
+      python_version: ${{ matrix.python3_version }}

.github/workflows/platform-integration-test.yaml

@@ -12,6 +12,10 @@ on:
         description: The Python wheel to test
         required: true
         type: string
+      python_version:
+        description: The wheel's Python version (like "3.12" or "3.13")
+        required: true
+        type: string
 
 permissions:
   contents: read
@@ -25,8 +29,8 @@ jobs:
 
       - uses: actions/cache/restore@v4
         with:
-          path: dist/otdf_python-0.1.10-py3-none-any.whl
-          key: ${{ runner.os }}-data-${{ github.sha }}
+          path: dist/otdf_python-0.1.11-py3-none-any.whl
+          key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }}
 
       - name: Prove that the input file is available
         shell: bash
@@ -112,7 +116,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-            python-version: '3.12'
+            python-version: ${{ inputs.python_version }}
 
       - name: Validate the Python SDK
         env:

.pre-commit-config.yaml

@@ -40,7 +40,7 @@ repos:
 
     - repo: https://github.com/astral-sh/ruff-pre-commit
       # Ruff version.
-      rev: v0.7.4
+      rev: v0.8.0
       hooks:
         # Run the linter.
         - id: ruff

ci-build.sh

@@ -2,6 +2,37 @@
 
 set -eou pipefail
 
+printf """
+
+✨✨✨ Configure gopy / dependencies, and build wheel ✨✨✨
+
+"""
+
+echo "python -VV"
+python -VV
+
+echo 'python -c "import sys; print(sys.executable)"'
+python -c "import sys; print(sys.executable)"
+
+echo 'pip -V'
+pip -V
+
+echo 'pip install poetry'
+pip install poetry
+
+# Look for go/bin (skip, we know it exists)
+echo '$HOME/'
+ls -la "$HOME/"
+
+echo '$HOME/.local/'
+ls -la "$HOME/.local/"
+
+echo '$HOME/go/'
+ls -la "$HOME/go/"
+
+# Add Go bin directory to PATH
+echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV
+
 # Since we don't have our wheel build / install configured yet we use '--no-root'
 poetry install --no-root
 
@@ -19,3 +50,5 @@ poetry run pip install --upgrade setuptools wheel
 gopy build --output=otdf_python -vm=python3 .
 
 poetry run python3 setup.py bdist_wheel
+
+pip install dist/otdf_python-0.1.11-py3-none-any.whl

make_and_validate_script.sh

@@ -42,7 +42,7 @@ python3 -m pip install --upgrade setuptools wheel
 python3 setup.py bdist_wheel
 
 # Prove that the wheel can be installed
-pip install dist/otdf_python-0.1.10-py3-none-any.whl
+pip install dist/otdf_python-0.1.11-py3-none-any.whl
 
 if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then
     echo "Build is complete, skipping tests."