From 26bc95bdc0396e12563a952d3d5f0dd9142775fe Mon Sep 17 00:00:00 2001 From: b-long Date: Wed, 9 Oct 2024 21:06:09 -0400 Subject: [PATCH 1/8] Create Composite Action for integration testing --- .github/workflows/build-golang-macos.yaml | 4 ++ .github/workflows/build-golang-ubuntu.yaml | 61 ++++++------------- .../workflows/platform-integration-test.yaml | 50 ++++++++++----- 3 files changed, 57 insertions(+), 58 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index 21eb22e..deb23d3 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -102,6 +102,10 @@ jobs: # Test wheel functionality # python3 validate_otdf_python.py + - uses: ./.github/workflows/platform-integration-test.yaml + with: + wheel: dist/otdf_python-0.0.15-py3-none-any.whl + # release: # needs: build # runs-on: macos-latest diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index 475b5d6..b7653be 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -75,55 +75,12 @@ jobs: poetry run python3 setup.py bdist_wheel - # TODO: Can this be a distinct job? - # exercise-ssh: - # needs: build - - # runs-on: ubuntu-latest - - # steps: - # - uses: actions/checkout@v4 - name: Check existing workspace content run: | ls -la ls -la dist/ - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - - # - name: Install Vagrant - # run: sudo apt-get update && sudo apt-get install -y vagrant - - # - name: Create Vagrantfile - # run: | - # cat << EOF > Vagrantfile - # Vagrant.configure("2") do |config| - # config.vm.provider "docker" do |d| - # d.image = "ubuntu" - # d.cmd = [ "/bin/bash", "-c", "echo 'Hello world'" ] - # d.remains_running = false - # end - # end - # EOF - - # - name: Run vagrant up - # run: vagrant up --provider=docker - - # - name: Launch the Docker/Vagrant test machine - # run: | - # # Change directory - # cd ssh-servers/docker-in-vagrant - - # # List content in the directory - # ls -la - - # # Launch machine - # vagrant up --provider=docker - - # # Connect to it with SSH - # vagrant ssh -c "echo 'Hello Ubuntu'" - - name: Test Python wheel run: | # Test wheel installation @@ -132,3 +89,21 @@ jobs: # DISABLED: Need to figure out Ubuntu nested VM # Test wheel functionality # python3 validate_otdf_python.py + + - uses: actions/cache/restore@v4 + with: + path: dist/otdf_python-0.0.15-py3-none-any.whl + key: ${{ runner.os }}-data-${{ github.sha }} + + - uses: actions/cache/save@v4 + with: + path: dist/otdf_python-0.0.15-py3-none-any.whl + key: ${{ runner.os }}-data-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-data- + + integration-test: + needs: build + uses: ./.github/workflows/platform-integration-test.yaml + with: + wheel: dist/otdf_python-0.0.15-py3-none-any.whl diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index 7988c15..a92d615 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -1,31 +1,43 @@ # Based on # https://github.com/opentdf/java-sdk/blob/v0.6.1/.github/workflows/checks.yaml +# +# Except, that this is a "Composite Action", and specifies 'shell: bash' for +# each 'run:' step. name: "Platform Integration testing" on: - pull_request: - -# on: -# pull_request: -# branches: -# - main -# push: -# branches: -# - main + workflow_call: + inputs: + wheel: + description: The Python wheel to test + required: true + type: string permissions: contents: read jobs: - - platform-integration: - runs-on: ubuntu-22.04 + integration_test: + runs-on: ubuntu-latest steps: - - name: Checkout Java SDK + - name: Checkout this repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa + + - uses: actions/cache/restore@v4 with: - github_token: ${{ secrets.GITHUB_TOKEN }} + path: dist/otdf_python-0.0.15-py3-none-any.whl + key: ${{ runner.os }}-data-${{ github.sha }} + + - name: Prove that the input file is available + shell: bash + run: | + ls -la + ls -la "${{ inputs.wheel }}" + du -sh "${{ inputs.wheel }}" + + # - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa + # with: + # github_token: ${{ secrets.GITHUB_TOKEN }} - name: Check out platform uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 @@ -44,10 +56,13 @@ jobs: platform/protocol/go/go.sum platform/sdk/go.sum - run: go mod download + shell: bash working-directory: platform - run: go mod verify + shell: bash working-directory: platform - name: Create keys + shell: bash run: | .github/scripts/init-temp-keys.sh cp opentdf-dev.yaml opentdf.yaml @@ -64,12 +79,15 @@ jobs: # -alias localhost-for-tests # working-directory: platform/keys - name: Bring the services up + shell: bash run: docker compose up -d --wait --wait-timeout 240 working-directory: platform - name: Provision keycloak + shell: bash run: go run ./service provision keycloak working-directory: platform - name: Provision fixtures + shell: bash run: go run ./service provision fixtures working-directory: platform - name: Start server in background @@ -83,8 +101,10 @@ jobs: wait-for: 90s working-directory: platform - name: Get grpcurl + shell: bash run: go install github.com/fullstorydev/grpcurl/cmd/grpcurl@v1.8.9 - name: Make sure that the platform is up + shell: bash run: | grpcurl -plaintext localhost:8080 list && \ grpcurl -plaintext localhost:8080 kas.AccessService/PublicKey From 1fae5e279b79382d3f92f7c17942cffd73a3fd6b Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 20:57:32 -0500 Subject: [PATCH 2/8] Run 'pre-commit autoupdate' --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7db081a..cd1a8fc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ exclude: | # See https://pre-commit.com/hooks.html for more hooks# repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-yaml - id: end-of-file-fixer @@ -40,7 +40,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.6.9 + rev: v0.7.4 hooks: # Run the linter. - id: ruff From 7870aaaa6da43c9d74ebb6d36f04e0ad47eceac1 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 20:58:15 -0500 Subject: [PATCH 3/8] Config using env vars --- validate_otdf_python.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/validate_otdf_python.py b/validate_otdf_python.py index a124928..a2e5fb9 100644 --- a/validate_otdf_python.py +++ b/validate_otdf_python.py @@ -5,6 +5,7 @@ import tempfile from pathlib import Path from zipfile import is_zipfile +from os import environ from otdf_python.gotdf_python import EncryptionConfig @@ -22,11 +23,14 @@ def _get_configuration() -> EncryptionConfig: platformEndpoint = "localhost:8080" config: EncryptionConfig = EncryptionConfig( - ClientId="opentdf-sdk", - ClientSecret="secret", - PlatformEndpoint=platformEndpoint, - TokenEndpoint="http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token", - KasUrl=f"http://{platformEndpoint}/kas", + ClientId=environ.get("OPENTDF_CLIENT_ID", "opentdf-sdk"), + ClientSecret=environ.get("OPENTDF_CLIENT_SECRET", "secret"), + PlatformEndpoint=environ.get("OPENTDF_HOSTNAME", platformEndpoint), + TokenEndpoint=environ.get( + "OIDC_TOKEN_ENDPOINT", + "http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token", + ), + KasUrl=environ.get("OPENTDF_KAS_URL", f"http://{platformEndpoint}/kas"), # FIXME: Be careful with binding the 'DataAttributes' field on this struct. # # In golang, this is initialized as []string , but passing From 1b1886cb2c973006d2f5a78a57b673f1e53fb783 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 21:06:16 -0500 Subject: [PATCH 4/8] Run 'validate_otdf_python.py' in CI --- .../workflows/platform-integration-test.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index a92d615..a8f158f 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -109,6 +109,24 @@ jobs: grpcurl -plaintext localhost:8080 list && \ grpcurl -plaintext localhost:8080 kas.AccessService/PublicKey + - name: Validate the Python SDK + env: + OPENTDF_CLIENT_ID: "opentdf-sdk" + OPENTDF_CLIENT_SECRET: "secret" + OPENTDF_HOSTNAME: "localhost:8080" + OIDC_TOKEN_ENDPOINT: "http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token" + OPENTDF_KAS_URL: "http://localhost:8080/kas" + run: | + mkdir validation + wheel="$(basename ${{ inputs.wheel }} )" + cp -v "${{ inputs.wheel }}" validation/ + cp -v validate_otdf_python.py validation/ + cd validation + python -m venv .venv + source .venv/bin/activate + pip install ./"$wheel" + python validate_otdf_python.py + # - name: Validate the SDK through the command line interface # run: | # printf 'here is some data to encrypt' > data From d2d702be6759c6d0fae751869836dcf724061792 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 21:55:20 -0500 Subject: [PATCH 5/8] DRY ci-build.sh --- .github/workflows/build-golang-macos.yaml | 18 +----------------- .github/workflows/build-golang-ubuntu.yaml | 19 +------------------ ci-build.sh | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 35 deletions(-) create mode 100755 ci-build.sh diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index deb23d3..e197869 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -76,23 +76,7 @@ jobs: # FIXME: Add more caching - name: Add gopy dependencies and build wheel run: | - # Since we don't have our wheel build / install configured yet we use '--no-root' - poetry install --no-root - - source $(poetry env info --path)/bin/activate - - # Add Go bin directory to PATH - echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV - - go install golang.org/x/tools/cmd/goimports@latest - - go install github.com/go-python/gopy@latest - - poetry run pip install --upgrade setuptools wheel - - gopy build --output=otdf_python -vm=python3 . - - poetry run python3 setup.py bdist_wheel + ./ci-build.sh - name: Test Python wheel run: | diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index b7653be..b4424ba 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -57,24 +57,7 @@ jobs: # FIXME: Add more caching - name: Add gopy dependencies and build wheel run: | - # Since we don't have our wheel build / install configured yet we use '--no-root' - poetry install --no-root - - source $(poetry env info --path)/bin/activate - - # Add Go bin directory to PATH - echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV - - go install golang.org/x/tools/cmd/goimports@latest - - go install github.com/go-python/gopy@latest - - poetry run pip install --upgrade setuptools wheel - - gopy build --output=otdf_python -vm=python3 . - - poetry run python3 setup.py bdist_wheel - + ./ci-build.sh - name: Check existing workspace content run: | diff --git a/ci-build.sh b/ci-build.sh new file mode 100755 index 0000000..30dffc5 --- /dev/null +++ b/ci-build.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -eou pipefail + +# Since we don't have our wheel build / install configured yet we use '--no-root' +poetry install --no-root + +source $(poetry env info --path)/bin/activate + +# Add Go bin directory to PATH +echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV + +go install golang.org/x/tools/cmd/goimports@latest + +go install github.com/go-python/gopy@latest + +poetry run pip install --upgrade setuptools wheel + +gopy build --output=otdf_python -vm=python3 . + +poetry run python3 setup.py bdist_wheel From 7abe4cbf6712b40befa15942014b4acb7fa65b62 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 21:59:49 -0500 Subject: [PATCH 6/8] Run 'validate_otdf_python.py' in CI --- .github/workflows/platform-integration-test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index a8f158f..6217db5 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -109,6 +109,11 @@ jobs: grpcurl -plaintext localhost:8080 list && \ grpcurl -plaintext localhost:8080 kas.AccessService/PublicKey + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Validate the Python SDK env: OPENTDF_CLIENT_ID: "opentdf-sdk" From 850ddb804ceb05367dad0079fd8d7a3e5e4ebd40 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 22:10:32 -0500 Subject: [PATCH 7/8] Improve 'validate_otdf_python.py' --- validate_otdf_python.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/validate_otdf_python.py b/validate_otdf_python.py index a2e5fb9..6ef8b68 100644 --- a/validate_otdf_python.py +++ b/validate_otdf_python.py @@ -9,8 +9,6 @@ from otdf_python.gotdf_python import EncryptionConfig -SOME_PLAINTEXT_FILE = Path(__file__).parent / "go.mod" - def verify_hello(): from otdf_python.gotdf_python import Hello @@ -81,6 +79,9 @@ def verify_encrypt_file() -> None: "The output path should not exist before calling 'EncryptFile()'." ) + SOME_PLAINTEXT_FILE = Path(tmpDir) / "new-file.txt" + SOME_PLAINTEXT_FILE.write_text("Hello world") + outputFilePath = EncryptFile( inputFilePath=str(SOME_PLAINTEXT_FILE), outputFilePath=str(SOME_ENCRYPTED_FILE), @@ -91,10 +92,10 @@ def verify_encrypt_file() -> None: if not SOME_ENCRYPTED_FILE.exists(): raise ValueError("The output file does not exist!") - if not ( - SOME_ENCRYPTED_FILE.stat().st_size > 2500 - and is_zipfile(SOME_ENCRYPTED_FILE) - ): + encrypted_file_size = SOME_ENCRYPTED_FILE.stat().st_size + print(f"The encrypted file size is {encrypted_file_size}") + + if not (encrypted_file_size > 1500 and is_zipfile(SOME_ENCRYPTED_FILE)): raise ValueError("The output file has unexpected content!") # breakpoint() From de88dd370f2e07f16f1c527527c7176ff01856d9 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 22:25:13 -0500 Subject: [PATCH 8/8] Bump 0.0.15 -> 0.1.0 --- .github/workflows/build-golang-macos.yaml | 4 ++-- .github/workflows/build-golang-ubuntu.yaml | 8 ++++---- .github/workflows/platform-integration-test.yaml | 2 +- pyproject.toml | 2 +- setup.py | 2 +- setup_ci.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index e197869..3849a70 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -81,14 +81,14 @@ jobs: - name: Test Python wheel run: | # Test wheel installation - pip install dist/otdf_python-0.0.15-py3-none-any.whl + pip install dist/otdf_python-0.1.0-py3-none-any.whl # Test wheel functionality # python3 validate_otdf_python.py - uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.0.15-py3-none-any.whl + wheel: dist/otdf_python-0.1.0-py3-none-any.whl # release: # needs: build diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index b4424ba..d9e2083 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -67,7 +67,7 @@ jobs: - name: Test Python wheel run: | # Test wheel installation - pip install dist/otdf_python-0.0.15-py3-none-any.whl + pip install dist/otdf_python-0.1.0-py3-none-any.whl # DISABLED: Need to figure out Ubuntu nested VM # Test wheel functionality @@ -75,12 +75,12 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.0.15-py3-none-any.whl + path: dist/otdf_python-0.1.0-py3-none-any.whl key: ${{ runner.os }}-data-${{ github.sha }} - uses: actions/cache/save@v4 with: - path: dist/otdf_python-0.0.15-py3-none-any.whl + path: dist/otdf_python-0.1.0-py3-none-any.whl key: ${{ runner.os }}-data-${{ github.sha }} restore-keys: | ${{ runner.os }}-data- @@ -89,4 +89,4 @@ jobs: needs: build uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.0.15-py3-none-any.whl + wheel: dist/otdf_python-0.1.0-py3-none-any.whl diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index 6217db5..e1627aa 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -25,7 +25,7 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.0.15-py3-none-any.whl + path: dist/otdf_python-0.1.0-py3-none-any.whl key: ${{ runner.os }}-data-${{ github.sha }} - name: Prove that the input file is available diff --git a/pyproject.toml b/pyproject.toml index c805719..375f3c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "otdf-python" # Should match 'setup.py' version number (used for gopy/pybindgen) -version = "0.0.15" +version = "0.1.0" description = "Unofficial OpenTDF SDK for Python." authors = ["b-long "] readme = "README.md" diff --git a/setup.py b/setup.py index a1e478e..7954b32 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url="https://github.com/b-long/opentdf-python-sdk", package_data={"otdf_python": ["*.so"]}, # Should match 'pyproject.toml' version number - version="0.0.15", + version="0.1.0", author_email="b-long@users.noreply.github.com", include_package_data=True, ) diff --git a/setup_ci.py b/setup_ci.py index 26e3e3b..3e68983 100644 --- a/setup_ci.py +++ b/setup_ci.py @@ -81,7 +81,7 @@ def build_extension(self, ext: Extension): setuptools.setup( name="otdf_python", - version="0.0.15", + version="0.1.0", author="b-long", description="Unofficial OpenTDF SDK for Python.", long_description_content_type="text/markdown",