From 1742f85123a660a47b9051dfafd02326451aebd6 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 11 Feb 2025 17:15:34 -0500 Subject: [PATCH 1/8] Initialize with a size for speed --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index c847dea..542814d 100644 --- a/main.go +++ b/main.go @@ -439,8 +439,8 @@ func EncryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O return nil, err } - var outputPaths []string - var errors []error + var outputPaths = make([]string, 0, len(files)) + var errors = make([]error, 0, len(files)) for _, file := range files { inputFilePath := file outputFilePath := inputFilePath + ".tdf" From 4bff87ec03d06c88a3047eb4aef5e689051bf66d Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 11 Feb 2025 17:16:25 -0500 Subject: [PATCH 2/8] Provide more error detail --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 542814d..8de7d89 100644 --- a/main.go +++ b/main.go @@ -726,7 +726,7 @@ func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O logOutputPaths(outputPaths, errors) if len(errors) > 0 { - return outputPaths, fmt.Errorf("encountered errors during decryption: %v", errors) + return outputPaths, fmt.Errorf("encountered errors during decryption of files in directory %s: %v", dirPath, errors) } return outputPaths, nil } From 2faddcbece080e76d8e870519d74aa649a8ff2e4 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 11 Feb 2025 17:18:00 -0500 Subject: [PATCH 3/8] Provide comment for 'logOutputPaths' --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 8de7d89..95639f7 100644 --- a/main.go +++ b/main.go @@ -843,6 +843,7 @@ func findFiles(dir string, extensions []string) ([]string, error) { return files, nil } +// logOutputPaths logs the output paths and any errors that occurred during processing func logOutputPaths(outputPaths []string, errors []error) { if len(errors) > 0 { log.Println("Errors occurred during processing:") From 4eab9cfc4110a6159e5b9e181d86421509dc8b4d Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 11 Feb 2025 17:30:45 -0500 Subject: [PATCH 4/8] Improve return values/logic --- main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 95639f7..748a259 100644 --- a/main.go +++ b/main.go @@ -725,8 +725,11 @@ func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O logOutputPaths(outputPaths, errors) - if len(errors) > 0 { - return outputPaths, fmt.Errorf("encountered errors during decryption of files in directory %s: %v", dirPath, errors) + if len(outputPaths) == 0 { + if len(errors) == 0 { + return nil, fmt.Errorf("no files with extensions %v found in directory %s", extensions, dirPath) + } + return nil, fmt.Errorf("encountered errors during decryption of files in directory %s: %v", dirPath, errors) } return outputPaths, nil } From 44ff880969442807861b9fcc807f2c6b2178f60c Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 11 Feb 2025 17:32:01 -0500 Subject: [PATCH 5/8] Initialize with a size for speed --- main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 748a259..446f145 100644 --- a/main.go +++ b/main.go @@ -683,8 +683,9 @@ func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O return nil, err } - var outputPaths []string - var errors []error + var outputPaths = make([]string, 0, len(files)) + var errors = make([]error, 0, len(files)) + for _, file := range files { if !file.IsDir() { for _, ext := range extensions { From 2ba4f60c80f4fdc7d68bcf4585d4027612cedfe7 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 11 Feb 2025 17:39:11 -0500 Subject: [PATCH 6/8] Use newSdkClient in loop & use channels for data --- main.go | 92 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/main.go b/main.go index 446f145..5010cc3 100644 --- a/main.go +++ b/main.go @@ -673,57 +673,79 @@ in the same directory as the input files, with the .tdf extension removed from t */ func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config OpentdfConfig) ([]string, error) { authScopes := []string{"email"} - sdkClient, err := newSdkClient(config, authScopes) - if err != nil { - return nil, err - } files, err := os.ReadDir(dirPath) if err != nil { return nil, err } - var outputPaths = make([]string, 0, len(files)) - var errors = make([]error, 0, len(files)) + outputPathsChan := make(chan string, len(files)) + errChan := make(chan error, len(files)) + + var wg sync.WaitGroup for _, file := range files { if !file.IsDir() { for _, ext := range extensions { if strings.HasSuffix(file.Name(), ext) { - inputFilePath := filepath.Join(dirPath, file.Name()) - outputFilePath := strings.TrimSuffix(inputFilePath, ext) - - bytes, err := readBytesFromFile(inputFilePath) - if err != nil { - errors = append(errors, fmt.Errorf("failed to read file %s: %v", inputFilePath, err)) - continue - } - - decrypted, err := decryptBytesWithClient(bytes, sdkClient) - if err != nil { - errors = append(errors, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err)) - continue - } - - tdfFile, err := os.Create(outputFilePath) - if err != nil { - errors = append(errors, fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err)) - continue - } - defer tdfFile.Close() - - _, e := io.Copy(tdfFile, decrypted) - if e != nil { - errors = append(errors, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err)) - continue - } - - outputPaths = append(outputPaths, outputFilePath) + wg.Add(1) + go func(file os.DirEntry, ext string) { + defer wg.Done() + sdkClient, err := newSdkClient(config, authScopes) + if err != nil { + errChan <- fmt.Errorf("failed to create SDK client: %v", err) + return + } + + inputFilePath := filepath.Join(dirPath, file.Name()) + outputFilePath := strings.TrimSuffix(inputFilePath, ext) + + bytes, err := readBytesFromFile(inputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to read file %s: %v", inputFilePath, err) + return + } + + decrypted, err := decryptBytesWithClient(bytes, sdkClient) + if err != nil { + errChan <- fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err) + return + } + + tdfFile, err := os.Create(outputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err) + return + } + defer tdfFile.Close() + + _, e := io.Copy(tdfFile, decrypted) + if e != nil { + errChan <- fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err) + return + } + + outputPathsChan <- outputFilePath + }(file, ext) } } } } + wg.Wait() + close(outputPathsChan) + close(errChan) + + var outputPaths []string + for path := range outputPathsChan { + outputPaths = append(outputPaths, path) + } + + var errors []error + for err := range errChan { + errors = append(errors, err) + } + logOutputPaths(outputPaths, errors) if len(outputPaths) == 0 { From c95e1dcce65b2315a8060ea1700f0683da62c84f Mon Sep 17 00:00:00 2001 From: b-long Date: Wed, 12 Feb 2025 09:25:34 -0500 Subject: [PATCH 7/8] Add timeout to ARM builds --- .github/workflows/publish-test.yaml | 1 + .github/workflows/publish.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/publish-test.yaml b/.github/workflows/publish-test.yaml index 7cb73c3..af9bebb 100644 --- a/.github/workflows/publish-test.yaml +++ b/.github/workflows/publish-test.yaml @@ -127,6 +127,7 @@ jobs: build_linux_arm: name: Linux Python ARM runs-on: ubuntu-22.04 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 457183a..ca73a26 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -127,6 +127,7 @@ jobs: build_linux_arm: name: Linux Python ARM runs-on: ubuntu-22.04 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 From 18525f65a99d4c9bd7da6d516837fe7e652114e4 Mon Sep 17 00:00:00 2001 From: b-long Date: Wed, 12 Feb 2025 09:26:26 -0500 Subject: [PATCH 8/8] Bump 0.2.8 -> 0.2.9 --- .github/workflows/build-golang-macos.yaml | 2 +- .github/workflows/build-golang-ubuntu.yaml | 6 +++--- .github/workflows/platform-integration-test.yaml | 2 +- README.md | 4 ++-- build-scripts/ci-build.sh | 2 +- build-scripts/make_and_validate_script.sh | 2 +- build-scripts/uv_make_and_validate_script.sh | 2 +- go.sum | 4 ++-- pyproject.toml | 4 ++-- setup.py | 2 +- setup_ci.py | 2 +- uv.lock | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index 86ad73b..31a2e25 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -47,4 +47,4 @@ jobs: # - uses: ./.github/workflows/platform-integration-test.yaml # with: - # wheel: dist/otdf_python-0.2.8-py3-none-any.whl + # wheel: dist/otdf_python-0.2.9-py3-none-any.whl diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index 2a004cd..5e1c0db 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -43,12 +43,12 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.8-py3-none-any.whl + path: dist/otdf_python-0.2.9-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} - uses: actions/cache/save@v4 with: - path: dist/otdf_python-0.2.8-py3-none-any.whl + path: dist/otdf_python-0.2.9-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} restore-keys: | ${{ runner.os }}${{ matrix.python3_version }}-data- @@ -61,5 +61,5 @@ jobs: needs: build uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.2.8-py3-none-any.whl + wheel: dist/otdf_python-0.2.9-py3-none-any.whl python_version: ${{ matrix.python3_version }} diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index a042053..1d7b5d0 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.8-py3-none-any.whl + path: dist/otdf_python-0.2.9-py3-none-any.whl key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }} - name: Prove that the input file is available diff --git a/README.md b/README.md index a4eb971..42c96eb 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,10 @@ Install from the [Python Package Index (PyPI)](https://pypi.org): pip install otdf_python # Install a pinned version -pip install otdf-python==0.2.8 +pip install otdf-python==0.2.9 # Install a pinned version, from test.pypi.org -pip install -i https://test.pypi.org/simple/ otdf-python==0.2.8 +pip install -i https://test.pypi.org/simple/ otdf-python==0.2.9 ``` ## Usage diff --git a/build-scripts/ci-build.sh b/build-scripts/ci-build.sh index 8158e11..4584290 100755 --- a/build-scripts/ci-build.sh +++ b/build-scripts/ci-build.sh @@ -72,4 +72,4 @@ echo "✨✨✨ Build wheel" poetry run python3 setup.py bdist_wheel echo "✨✨✨ Install wheel" -pip install dist/otdf_python-0.2.8-py3-none-any.whl +pip install dist/otdf_python-0.2.9-py3-none-any.whl diff --git a/build-scripts/make_and_validate_script.sh b/build-scripts/make_and_validate_script.sh index b49c8f1..62f6b83 100755 --- a/build-scripts/make_and_validate_script.sh +++ b/build-scripts/make_and_validate_script.sh @@ -47,7 +47,7 @@ python3 -m pip install --upgrade setuptools wheel python3 setup.py bdist_wheel # Prove that the wheel can be installed -pip install dist/otdf_python-0.2.8-py3-none-any.whl +pip install dist/otdf_python-0.2.9-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/build-scripts/uv_make_and_validate_script.sh b/build-scripts/uv_make_and_validate_script.sh index 2d2a374..e9f6ab6 100755 --- a/build-scripts/uv_make_and_validate_script.sh +++ b/build-scripts/uv_make_and_validate_script.sh @@ -70,7 +70,7 @@ loud_print "Installing wheel" uv venv .venv-wheel --python 3.12 "$PY_TYPE" source "${BUILD_ROOT}/.venv-wheel/bin/activate" pip install pybindgen -pip install dist/otdf_python-0.2.8-py3-none-any.whl +pip install dist/otdf_python-0.2.9-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/go.sum b/go.sum index 33b6f01..4d08aea 100644 --- a/go.sum +++ b/go.sum @@ -99,8 +99,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opentdf/platform/lib/fixtures v0.2.8 h1:lGYrMnbORtU62lxsJi8qPsxjFuNIkc4Dop8rVkH6pD0= -github.com/opentdf/platform/lib/fixtures v0.2.8/go.mod h1:8yCSe+oUzW9jbM573r9qgE68rjwDMNzktObiGVsO/W8= +github.com/opentdf/platform/lib/fixtures v0.2.9 h1:lGYrMnbORtU62lxsJi8qPsxjFuNIkc4Dop8rVkH6pD0= +github.com/opentdf/platform/lib/fixtures v0.2.9/go.mod h1:8yCSe+oUzW9jbM573r9qgE68rjwDMNzktObiGVsO/W8= github.com/opentdf/platform/lib/ocrypto v0.1.7 h1:IcCYRrwmMqntqUE8frmUDg5EZ0WMdldpGeGhbv9+/A8= github.com/opentdf/platform/lib/ocrypto v0.1.7/go.mod h1:4bhKPbRFzURMerH5Vr/LlszHvcoXQbfJXa0bpY7/7yg= github.com/opentdf/platform/protocol/go v0.2.22 h1:C/jjtwu5yTon8g0ewuN29QE7VXSQHyb2dx9W0U6Oqok= diff --git a/pyproject.toml b/pyproject.toml index 676d80f..3081133 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "otdf-python" # Should match 'setup.py' version number (used for gopy/pybindgen) -version = "0.2.8" +version = "0.2.9" description = "Unofficial OpenTDF SDK for Python." authors = [ {name="b-long", email="b-long@users.noreply.github.com"} @@ -19,7 +19,7 @@ pybindgen = "^0.22.1" [tool.poetry] package-mode = false -version = "0.2.8" +version = "0.2.9" [tool.poetry.dependencies] python = ">=3.11,<3.14" diff --git a/setup.py b/setup.py index 95dc039..a140562 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url="https://github.com/b-long/opentdf-python-sdk", package_data={"otdf_python": ["*.so"]}, # Should match 'pyproject.toml' version number - version="0.2.8", + version="0.2.9", author_email="b-long@users.noreply.github.com", include_package_data=True, ) diff --git a/setup_ci.py b/setup_ci.py index 5bb75af..bfac080 100644 --- a/setup_ci.py +++ b/setup_ci.py @@ -81,7 +81,7 @@ def build_extension(self, ext: Extension): setuptools.setup( name="otdf_python", - version="0.2.8", + version="0.2.9", author="b-long", description="Unofficial OpenTDF SDK for Python.", long_description_content_type="text/markdown", diff --git a/uv.lock b/uv.lock index 138c19b..59938cf 100644 --- a/uv.lock +++ b/uv.lock @@ -3,5 +3,5 @@ requires-python = ">=3.11" [[package]] name = "otdf-python" -version = "0.2.8" +version = "0.2.9" source = { editable = "." }