From 1fae5e279b79382d3f92f7c17942cffd73a3fd6b Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 20:57:32 -0500 Subject: [PATCH 1/7] Run 'pre-commit autoupdate' --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7db081a..cd1a8fc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ exclude: | # See https://pre-commit.com/hooks.html for more hooks# repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-yaml - id: end-of-file-fixer @@ -40,7 +40,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.6.9 + rev: v0.7.4 hooks: # Run the linter. - id: ruff From 7870aaaa6da43c9d74ebb6d36f04e0ad47eceac1 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 20:58:15 -0500 Subject: [PATCH 2/7] Config using env vars --- validate_otdf_python.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/validate_otdf_python.py b/validate_otdf_python.py index a124928..a2e5fb9 100644 --- a/validate_otdf_python.py +++ b/validate_otdf_python.py @@ -5,6 +5,7 @@ import tempfile from pathlib import Path from zipfile import is_zipfile +from os import environ from otdf_python.gotdf_python import EncryptionConfig @@ -22,11 +23,14 @@ def _get_configuration() -> EncryptionConfig: platformEndpoint = "localhost:8080" config: EncryptionConfig = EncryptionConfig( - ClientId="opentdf-sdk", - ClientSecret="secret", - PlatformEndpoint=platformEndpoint, - TokenEndpoint="http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token", - KasUrl=f"http://{platformEndpoint}/kas", + ClientId=environ.get("OPENTDF_CLIENT_ID", "opentdf-sdk"), + ClientSecret=environ.get("OPENTDF_CLIENT_SECRET", "secret"), + PlatformEndpoint=environ.get("OPENTDF_HOSTNAME", platformEndpoint), + TokenEndpoint=environ.get( + "OIDC_TOKEN_ENDPOINT", + "http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token", + ), + KasUrl=environ.get("OPENTDF_KAS_URL", f"http://{platformEndpoint}/kas"), # FIXME: Be careful with binding the 'DataAttributes' field on this struct. # # In golang, this is initialized as []string , but passing From 1b1886cb2c973006d2f5a78a57b673f1e53fb783 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 21:06:16 -0500 Subject: [PATCH 3/7] Run 'validate_otdf_python.py' in CI --- .../workflows/platform-integration-test.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index a92d615..a8f158f 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -109,6 +109,24 @@ jobs: grpcurl -plaintext localhost:8080 list && \ grpcurl -plaintext localhost:8080 kas.AccessService/PublicKey + - name: Validate the Python SDK + env: + OPENTDF_CLIENT_ID: "opentdf-sdk" + OPENTDF_CLIENT_SECRET: "secret" + OPENTDF_HOSTNAME: "localhost:8080" + OIDC_TOKEN_ENDPOINT: "http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token" + OPENTDF_KAS_URL: "http://localhost:8080/kas" + run: | + mkdir validation + wheel="$(basename ${{ inputs.wheel }} )" + cp -v "${{ inputs.wheel }}" validation/ + cp -v validate_otdf_python.py validation/ + cd validation + python -m venv .venv + source .venv/bin/activate + pip install ./"$wheel" + python validate_otdf_python.py + # - name: Validate the SDK through the command line interface # run: | # printf 'here is some data to encrypt' > data From d2d702be6759c6d0fae751869836dcf724061792 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 21:55:20 -0500 Subject: [PATCH 4/7] DRY ci-build.sh --- .github/workflows/build-golang-macos.yaml | 18 +----------------- .github/workflows/build-golang-ubuntu.yaml | 19 +------------------ ci-build.sh | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 35 deletions(-) create mode 100755 ci-build.sh diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index deb23d3..e197869 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -76,23 +76,7 @@ jobs: # FIXME: Add more caching - name: Add gopy dependencies and build wheel run: | - # Since we don't have our wheel build / install configured yet we use '--no-root' - poetry install --no-root - - source $(poetry env info --path)/bin/activate - - # Add Go bin directory to PATH - echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV - - go install golang.org/x/tools/cmd/goimports@latest - - go install github.com/go-python/gopy@latest - - poetry run pip install --upgrade setuptools wheel - - gopy build --output=otdf_python -vm=python3 . - - poetry run python3 setup.py bdist_wheel + ./ci-build.sh - name: Test Python wheel run: | diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index b7653be..b4424ba 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -57,24 +57,7 @@ jobs: # FIXME: Add more caching - name: Add gopy dependencies and build wheel run: | - # Since we don't have our wheel build / install configured yet we use '--no-root' - poetry install --no-root - - source $(poetry env info --path)/bin/activate - - # Add Go bin directory to PATH - echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV - - go install golang.org/x/tools/cmd/goimports@latest - - go install github.com/go-python/gopy@latest - - poetry run pip install --upgrade setuptools wheel - - gopy build --output=otdf_python -vm=python3 . - - poetry run python3 setup.py bdist_wheel - + ./ci-build.sh - name: Check existing workspace content run: | diff --git a/ci-build.sh b/ci-build.sh new file mode 100755 index 0000000..30dffc5 --- /dev/null +++ b/ci-build.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -eou pipefail + +# Since we don't have our wheel build / install configured yet we use '--no-root' +poetry install --no-root + +source $(poetry env info --path)/bin/activate + +# Add Go bin directory to PATH +echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV + +go install golang.org/x/tools/cmd/goimports@latest + +go install github.com/go-python/gopy@latest + +poetry run pip install --upgrade setuptools wheel + +gopy build --output=otdf_python -vm=python3 . + +poetry run python3 setup.py bdist_wheel From 7abe4cbf6712b40befa15942014b4acb7fa65b62 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 21:59:49 -0500 Subject: [PATCH 5/7] Run 'validate_otdf_python.py' in CI --- .github/workflows/platform-integration-test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index a8f158f..6217db5 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -109,6 +109,11 @@ jobs: grpcurl -plaintext localhost:8080 list && \ grpcurl -plaintext localhost:8080 kas.AccessService/PublicKey + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Validate the Python SDK env: OPENTDF_CLIENT_ID: "opentdf-sdk" From 850ddb804ceb05367dad0079fd8d7a3e5e4ebd40 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 22:10:32 -0500 Subject: [PATCH 6/7] Improve 'validate_otdf_python.py' --- validate_otdf_python.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/validate_otdf_python.py b/validate_otdf_python.py index a2e5fb9..6ef8b68 100644 --- a/validate_otdf_python.py +++ b/validate_otdf_python.py @@ -9,8 +9,6 @@ from otdf_python.gotdf_python import EncryptionConfig -SOME_PLAINTEXT_FILE = Path(__file__).parent / "go.mod" - def verify_hello(): from otdf_python.gotdf_python import Hello @@ -81,6 +79,9 @@ def verify_encrypt_file() -> None: "The output path should not exist before calling 'EncryptFile()'." ) + SOME_PLAINTEXT_FILE = Path(tmpDir) / "new-file.txt" + SOME_PLAINTEXT_FILE.write_text("Hello world") + outputFilePath = EncryptFile( inputFilePath=str(SOME_PLAINTEXT_FILE), outputFilePath=str(SOME_ENCRYPTED_FILE), @@ -91,10 +92,10 @@ def verify_encrypt_file() -> None: if not SOME_ENCRYPTED_FILE.exists(): raise ValueError("The output file does not exist!") - if not ( - SOME_ENCRYPTED_FILE.stat().st_size > 2500 - and is_zipfile(SOME_ENCRYPTED_FILE) - ): + encrypted_file_size = SOME_ENCRYPTED_FILE.stat().st_size + print(f"The encrypted file size is {encrypted_file_size}") + + if not (encrypted_file_size > 1500 and is_zipfile(SOME_ENCRYPTED_FILE)): raise ValueError("The output file has unexpected content!") # breakpoint() From de88dd370f2e07f16f1c527527c7176ff01856d9 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 19 Nov 2024 22:25:13 -0500 Subject: [PATCH 7/7] Bump 0.0.15 -> 0.1.0 --- .github/workflows/build-golang-macos.yaml | 4 ++-- .github/workflows/build-golang-ubuntu.yaml | 8 ++++---- .github/workflows/platform-integration-test.yaml | 2 +- pyproject.toml | 2 +- setup.py | 2 +- setup_ci.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index e197869..3849a70 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -81,14 +81,14 @@ jobs: - name: Test Python wheel run: | # Test wheel installation - pip install dist/otdf_python-0.0.15-py3-none-any.whl + pip install dist/otdf_python-0.1.0-py3-none-any.whl # Test wheel functionality # python3 validate_otdf_python.py - uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.0.15-py3-none-any.whl + wheel: dist/otdf_python-0.1.0-py3-none-any.whl # release: # needs: build diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index b4424ba..d9e2083 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -67,7 +67,7 @@ jobs: - name: Test Python wheel run: | # Test wheel installation - pip install dist/otdf_python-0.0.15-py3-none-any.whl + pip install dist/otdf_python-0.1.0-py3-none-any.whl # DISABLED: Need to figure out Ubuntu nested VM # Test wheel functionality @@ -75,12 +75,12 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.0.15-py3-none-any.whl + path: dist/otdf_python-0.1.0-py3-none-any.whl key: ${{ runner.os }}-data-${{ github.sha }} - uses: actions/cache/save@v4 with: - path: dist/otdf_python-0.0.15-py3-none-any.whl + path: dist/otdf_python-0.1.0-py3-none-any.whl key: ${{ runner.os }}-data-${{ github.sha }} restore-keys: | ${{ runner.os }}-data- @@ -89,4 +89,4 @@ jobs: needs: build uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.0.15-py3-none-any.whl + wheel: dist/otdf_python-0.1.0-py3-none-any.whl diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index 6217db5..e1627aa 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -25,7 +25,7 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.0.15-py3-none-any.whl + path: dist/otdf_python-0.1.0-py3-none-any.whl key: ${{ runner.os }}-data-${{ github.sha }} - name: Prove that the input file is available diff --git a/pyproject.toml b/pyproject.toml index c805719..375f3c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "otdf-python" # Should match 'setup.py' version number (used for gopy/pybindgen) -version = "0.0.15" +version = "0.1.0" description = "Unofficial OpenTDF SDK for Python." authors = ["b-long "] readme = "README.md" diff --git a/setup.py b/setup.py index a1e478e..7954b32 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url="https://github.com/b-long/opentdf-python-sdk", package_data={"otdf_python": ["*.so"]}, # Should match 'pyproject.toml' version number - version="0.0.15", + version="0.1.0", author_email="b-long@users.noreply.github.com", include_package_data=True, ) diff --git a/setup_ci.py b/setup_ci.py index 26e3e3b..3e68983 100644 --- a/setup_ci.py +++ b/setup_ci.py @@ -81,7 +81,7 @@ def build_extension(self, ext: Extension): setuptools.setup( name="otdf_python", - version="0.0.15", + version="0.1.0", author="b-long", description="Unofficial OpenTDF SDK for Python.", long_description_content_type="text/markdown",