From 7813f1cc912f08a328360108dc183cdecff02d18 Mon Sep 17 00:00:00 2001 From: b-long Date: Sun, 24 Nov 2024 14:29:00 -0500 Subject: [PATCH 1/5] DRY ci-build steps --- .github/workflows/build-golang-macos.yaml | 19 +------------ .github/workflows/build-golang-ubuntu.yaml | 19 +------------ ci-build.sh | 31 ++++++++++++++++++++++ 3 files changed, 33 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index 30b7e3b..55e884e 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -56,25 +56,8 @@ jobs: with: python-version: '3.12' - - name: Install dependencies - run: | - pip install poetry - - # Look for go/bin (skip, we know it exists) - echo '$HOME/' - ls -la "$HOME/" - - echo '$HOME/.local/' - ls -la "$HOME/.local/" - - echo '$HOME/go/' - ls -la "$HOME/go/" - - - name: Add Go bin directory to PATH - run: echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV - # FIXME: Add more caching - - name: Add gopy dependencies and build wheel + - name: Configure gopy / dependencies, and build wheel run: | ./ci-build.sh diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index 743a497..06234b9 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -37,25 +37,8 @@ jobs: with: python-version: '3.12' - - name: Install dependencies - run: | - pip install poetry - - # Look for go/bin (skip, we know it exists) - echo '$HOME/' - ls -la "$HOME/" - - echo '$HOME/.local/' - ls -la "$HOME/.local/" - - echo '$HOME/go/' - ls -la "$HOME/go/" - - - name: Add Go bin directory to PATH - run: echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV - # FIXME: Add more caching - - name: Add gopy dependencies and build wheel + - name: Configure gopy / dependencies, and build wheel run: | ./ci-build.sh diff --git a/ci-build.sh b/ci-build.sh index 30dffc5..087b714 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -2,6 +2,37 @@ set -eou pipefail +printf """ + +✨✨✨ Configure gopy / dependencies, and build wheel ✨✨✨ + +""" + +echo "python -VV" +python -VV + +echo 'python -c "import sys; print(sys.executable)"' +python -c "import sys; print(sys.executable)" + +echo 'pip -V' +pip -V + +echo 'pip install poetry' +pip install poetry + +# Look for go/bin (skip, we know it exists) +echo '$HOME/' +ls -la "$HOME/" + +echo '$HOME/.local/' +ls -la "$HOME/.local/" + +echo '$HOME/go/' +ls -la "$HOME/go/" + +# Add Go bin directory to PATH +echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV + # Since we don't have our wheel build / install configured yet we use '--no-root' poetry install --no-root From 55e8de3a0aa7fbfc7d387e069e45e29d2fe134bb Mon Sep 17 00:00:00 2001 From: b-long Date: Sun, 24 Nov 2024 14:29:49 -0500 Subject: [PATCH 2/5] Run 'pre-commit autoupdate' --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cd1a8fc..37cf25c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,7 +40,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.7.4 + rev: v0.8.0 hooks: # Run the linter. - id: ruff From c02ffcd11fd4222aa25677e027c6d28b0c365bfb Mon Sep 17 00:00:00 2001 From: b-long Date: Sun, 24 Nov 2024 14:41:27 -0500 Subject: [PATCH 3/5] DRY ci-build steps & remove unused publishing --- .github/workflows/build-golang-macos.yaml | 54 +--------------------- .github/workflows/build-golang-ubuntu.yaml | 16 ------- ci-build.sh | 2 + 3 files changed, 3 insertions(+), 69 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index 55e884e..fba499c 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -1,10 +1,7 @@ --- name: macOS -- Build Python wheel using golang's gopy -on: - push: - branches: - - main +on: [push] defaults: run: @@ -27,20 +24,6 @@ jobs: steps: - uses: actions/checkout@v4 - # - name: Setup Docker on macOS using Colima, Lima-VM, and Homebrew - # uses: douglascamata/setup-docker-macos-action@v1-alpha - - - # - name: Verify docker Installation - # run: | - # docker --version - # docker ps - - # - name: Setup vagrant - # run: | - # brew install vagrant - # vagrant --version - - name: Setup Go uses: actions/setup-go@v4 with: @@ -61,41 +44,6 @@ jobs: run: | ./ci-build.sh - - name: Test Python wheel - run: | - # Test wheel installation - pip install dist/otdf_python-0.1.10-py3-none-any.whl - - # Test wheel functionality - # python3 validate_otdf_python.py - # - uses: ./.github/workflows/platform-integration-test.yaml # with: # wheel: dist/otdf_python-0.1.10-py3-none-any.whl - - # release: - # needs: build - # runs-on: macos-latest - # if: github.ref == 'refs/heads/main' - # permissions: - # contents: write - # steps: - # - uses: actions/checkout@v4 - - - name: Store version - run: | - pip install poetry - - PROJECT_VESION=$(poetry version -s) - echo "PROJECT_VESION=$PROJECT_VESION" >> $GITHUB_ENV - - - name: Install twine - run: pip install twine - - - name: Upload to PyPI - env: - TWINE_UPLOAD_URL: https://upload.pypi.org/legacy/ - PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} - PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - twine upload --password "$PYPI_PASSWORD" --user "$PYPI_USERNAME" --repository-url "$TWINE_UPLOAD_URL" dist/* diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index 06234b9..61ca7ff 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -16,12 +16,10 @@ jobs: fail-fast: true matrix: go-version: [1.22.x] - # go-version: [ '1.19', '1.20', '1.22.x' ] steps: - uses: actions/checkout@v4 - - name: Setup Go uses: actions/setup-go@v4 with: @@ -42,20 +40,6 @@ jobs: run: | ./ci-build.sh - - name: Check existing workspace content - run: | - ls -la - ls -la dist/ - - - name: Test Python wheel - run: | - # Test wheel installation - pip install dist/otdf_python-0.1.10-py3-none-any.whl - - # DISABLED: Need to figure out Ubuntu nested VM - # Test wheel functionality - # python3 validate_otdf_python.py - - uses: actions/cache/restore@v4 with: path: dist/otdf_python-0.1.10-py3-none-any.whl diff --git a/ci-build.sh b/ci-build.sh index 087b714..2f32d74 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -50,3 +50,5 @@ poetry run pip install --upgrade setuptools wheel gopy build --output=otdf_python -vm=python3 . poetry run python3 setup.py bdist_wheel + +pip install dist/otdf_python-0.1.10-py3-none-any.whl From 79c352a9a80d4f683f12b0ffe931946ec4518355 Mon Sep 17 00:00:00 2001 From: b-long Date: Sun, 24 Nov 2024 21:18:41 -0500 Subject: [PATCH 4/5] Matrix python build --- .github/workflows/build-golang-macos.yaml | 61 +++++----- .github/workflows/build-golang-ubuntu.yaml | 114 +++++++++--------- .../workflows/platform-integration-test.yaml | 8 +- 3 files changed, 97 insertions(+), 86 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index fba499c..70cf741 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -1,5 +1,5 @@ --- -name: macOS -- Build Python wheel using golang's gopy +name: macOS build on: [push] @@ -14,36 +14,37 @@ jobs: build: runs-on: macos-13 permissions: - contents: write + contents: write strategy: - fail-fast: true - matrix: - go-version: [1.22.x] + fail-fast: true + matrix: + go-version: [1.22.x] + python3_version: [ "3.11", "3.12" ] steps: - - uses: actions/checkout@v4 - - - name: Setup Go - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: go.sum - - name: Install dependencies - run: go get . - - name: Test with Go - run: go test -timeout 40s -run ^TestHello$ gotdf_python -count=1 # go test - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' - - # FIXME: Add more caching - - name: Configure gopy / dependencies, and build wheel - run: | - ./ci-build.sh - - # - uses: ./.github/workflows/platform-integration-test.yaml - # with: - # wheel: dist/otdf_python-0.1.10-py3-none-any.whl + - uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go-version }} + cache-dependency-path: go.sum + - name: Install dependencies + run: go get . + - name: Test with Go + run: go test -timeout 40s -run ^TestHello$ gotdf_python -count=1 # go test + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python3_version }} + + # FIXME: Add more caching + - name: Configure gopy / dependencies, and build wheel + run: | + ./ci-build.sh + + # - uses: ./.github/workflows/platform-integration-test.yaml + # with: + # wheel: dist/otdf_python-0.1.10-py3-none-any.whl diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index 61ca7ff..cdfdf8c 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -1,59 +1,65 @@ --- -name: Ubuntu -- Build Python wheel using golang's gopy +name: Ubuntu build on: [push] jobs: - call-lint: - uses: ./.github/workflows/lint-on-ubuntu.yaml - - build: - runs-on: ubuntu-latest - permissions: - contents: write - - strategy: - fail-fast: true - matrix: - go-version: [1.22.x] - - steps: - - uses: actions/checkout@v4 - - - name: Setup Go - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: go.sum - - name: Install dependencies - run: go get . - - name: Test with Go - run: go test -timeout 40s -run ^TestHello$ gotdf_python -count=1 # go test - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' - - # FIXME: Add more caching - - name: Configure gopy / dependencies, and build wheel - run: | - ./ci-build.sh - - - uses: actions/cache/restore@v4 - with: - path: dist/otdf_python-0.1.10-py3-none-any.whl - key: ${{ runner.os }}-data-${{ github.sha }} - - - uses: actions/cache/save@v4 - with: - path: dist/otdf_python-0.1.10-py3-none-any.whl - key: ${{ runner.os }}-data-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-data- - - integration-test: - needs: build - uses: ./.github/workflows/platform-integration-test.yaml - with: - wheel: dist/otdf_python-0.1.10-py3-none-any.whl + call-lint: + uses: ./.github/workflows/lint-on-ubuntu.yaml + + build: + runs-on: ubuntu-latest + permissions: + contents: write + + strategy: + fail-fast: true + matrix: + go-version: [1.22.x] + python3_version: [ "3.11", "3.12" ] + + steps: + - uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go-version }} + cache-dependency-path: go.sum + - name: Install dependencies + run: go get . + - name: Test with Go + run: go test -timeout 40s -run ^TestHello$ gotdf_python -count=1 # go test + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python3_version }} + + # FIXME: Add more caching + - name: Configure gopy / dependencies, and build wheel + run: | + ./ci-build.sh + + - uses: actions/cache/restore@v4 + with: + path: dist/otdf_python-0.1.10-py3-none-any.whl + key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} + + - uses: actions/cache/save@v4 + with: + path: dist/otdf_python-0.1.10-py3-none-any.whl + key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} + restore-keys: | + ${{ runner.os }}${{ matrix.python3_version }}-data- + + integration-test: + strategy: + fail-fast: true + matrix: + python3_version: [ "3.11", "3.12" ] + needs: build + uses: ./.github/workflows/platform-integration-test.yaml + with: + wheel: dist/otdf_python-0.1.10-py3-none-any.whl + python_version: ${{ matrix.python3_version }} diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index 4a7e14b..06d3638 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -12,6 +12,10 @@ on: description: The Python wheel to test required: true type: string + python_version: + description: The wheel's Python version (like "3.12" or "3.13") + required: true + type: string permissions: contents: read @@ -26,7 +30,7 @@ jobs: - uses: actions/cache/restore@v4 with: path: dist/otdf_python-0.1.10-py3-none-any.whl - key: ${{ runner.os }}-data-${{ github.sha }} + key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }} - name: Prove that the input file is available shell: bash @@ -112,7 +116,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.12' + python-version: ${{ inputs.python_version }} - name: Validate the Python SDK env: From c0f536e6c4f971c8dfb9daff37473d534d8f17ce Mon Sep 17 00:00:00 2001 From: b-long Date: Mon, 9 Dec 2024 20:32:29 -0500 Subject: [PATCH 5/5] Bump 0.1.10 -> 0.1.11 --- .github/workflows/build-golang-macos.yaml | 2 +- .github/workflows/build-golang-ubuntu.yaml | 6 +++--- .github/workflows/platform-integration-test.yaml | 2 +- ci-build.sh | 2 +- make_and_validate_script.sh | 2 +- pyproject.toml | 2 +- setup.py | 2 +- setup_ci.py | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index 70cf741..3cfd9f5 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -47,4 +47,4 @@ jobs: # - uses: ./.github/workflows/platform-integration-test.yaml # with: - # wheel: dist/otdf_python-0.1.10-py3-none-any.whl + # wheel: dist/otdf_python-0.1.11-py3-none-any.whl diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index cdfdf8c..0e7bdf5 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -43,12 +43,12 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.1.10-py3-none-any.whl + path: dist/otdf_python-0.1.11-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} - uses: actions/cache/save@v4 with: - path: dist/otdf_python-0.1.10-py3-none-any.whl + path: dist/otdf_python-0.1.11-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} restore-keys: | ${{ runner.os }}${{ matrix.python3_version }}-data- @@ -61,5 +61,5 @@ jobs: needs: build uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.1.10-py3-none-any.whl + wheel: dist/otdf_python-0.1.11-py3-none-any.whl python_version: ${{ matrix.python3_version }} diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index 06d3638..e98358e 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.1.10-py3-none-any.whl + path: dist/otdf_python-0.1.11-py3-none-any.whl key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }} - name: Prove that the input file is available diff --git a/ci-build.sh b/ci-build.sh index 2f32d74..fc7384b 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -51,4 +51,4 @@ gopy build --output=otdf_python -vm=python3 . poetry run python3 setup.py bdist_wheel -pip install dist/otdf_python-0.1.10-py3-none-any.whl +pip install dist/otdf_python-0.1.11-py3-none-any.whl diff --git a/make_and_validate_script.sh b/make_and_validate_script.sh index d99b7c2..fd6a41f 100755 --- a/make_and_validate_script.sh +++ b/make_and_validate_script.sh @@ -42,7 +42,7 @@ python3 -m pip install --upgrade setuptools wheel python3 setup.py bdist_wheel # Prove that the wheel can be installed -pip install dist/otdf_python-0.1.10-py3-none-any.whl +pip install dist/otdf_python-0.1.11-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/pyproject.toml b/pyproject.toml index 3b60679..9fedb79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "otdf-python" # Should match 'setup.py' version number (used for gopy/pybindgen) -version = "0.1.10" +version = "0.1.11" description = "Unofficial OpenTDF SDK for Python." authors = ["b-long "] readme = "README.md" diff --git a/setup.py b/setup.py index 3fb1662..0e48a58 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url="https://github.com/b-long/opentdf-python-sdk", package_data={"otdf_python": ["*.so"]}, # Should match 'pyproject.toml' version number - version="0.1.10", + version="0.1.11", author_email="b-long@users.noreply.github.com", include_package_data=True, ) diff --git a/setup_ci.py b/setup_ci.py index 2d02efe..15df5cc 100644 --- a/setup_ci.py +++ b/setup_ci.py @@ -81,7 +81,7 @@ def build_extension(self, ext: Extension): setuptools.setup( name="otdf_python", - version="0.1.10", + version="0.1.11", author="b-long", description="Unofficial OpenTDF SDK for Python.", long_description_content_type="text/markdown",