diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index 86ad73b..31a2e25 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -47,4 +47,4 @@ jobs: # - uses: ./.github/workflows/platform-integration-test.yaml # with: - # wheel: dist/otdf_python-0.2.8-py3-none-any.whl + # wheel: dist/otdf_python-0.2.9-py3-none-any.whl diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index 2a004cd..5e1c0db 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -43,12 +43,12 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.8-py3-none-any.whl + path: dist/otdf_python-0.2.9-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} - uses: actions/cache/save@v4 with: - path: dist/otdf_python-0.2.8-py3-none-any.whl + path: dist/otdf_python-0.2.9-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} restore-keys: | ${{ runner.os }}${{ matrix.python3_version }}-data- @@ -61,5 +61,5 @@ jobs: needs: build uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.2.8-py3-none-any.whl + wheel: dist/otdf_python-0.2.9-py3-none-any.whl python_version: ${{ matrix.python3_version }} diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index a042053..1d7b5d0 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.8-py3-none-any.whl + path: dist/otdf_python-0.2.9-py3-none-any.whl key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }} - name: Prove that the input file is available diff --git a/.github/workflows/publish-test.yaml b/.github/workflows/publish-test.yaml index 7cb73c3..af9bebb 100644 --- a/.github/workflows/publish-test.yaml +++ b/.github/workflows/publish-test.yaml @@ -127,6 +127,7 @@ jobs: build_linux_arm: name: Linux Python ARM runs-on: ubuntu-22.04 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 457183a..ca73a26 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -127,6 +127,7 @@ jobs: build_linux_arm: name: Linux Python ARM runs-on: ubuntu-22.04 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index a4eb971..42c96eb 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,10 @@ Install from the [Python Package Index (PyPI)](https://pypi.org): pip install otdf_python # Install a pinned version -pip install otdf-python==0.2.8 +pip install otdf-python==0.2.9 # Install a pinned version, from test.pypi.org -pip install -i https://test.pypi.org/simple/ otdf-python==0.2.8 +pip install -i https://test.pypi.org/simple/ otdf-python==0.2.9 ``` ## Usage diff --git a/build-scripts/ci-build.sh b/build-scripts/ci-build.sh index 8158e11..4584290 100755 --- a/build-scripts/ci-build.sh +++ b/build-scripts/ci-build.sh @@ -72,4 +72,4 @@ echo "✨✨✨ Build wheel" poetry run python3 setup.py bdist_wheel echo "✨✨✨ Install wheel" -pip install dist/otdf_python-0.2.8-py3-none-any.whl +pip install dist/otdf_python-0.2.9-py3-none-any.whl diff --git a/build-scripts/make_and_validate_script.sh b/build-scripts/make_and_validate_script.sh index b49c8f1..62f6b83 100755 --- a/build-scripts/make_and_validate_script.sh +++ b/build-scripts/make_and_validate_script.sh @@ -47,7 +47,7 @@ python3 -m pip install --upgrade setuptools wheel python3 setup.py bdist_wheel # Prove that the wheel can be installed -pip install dist/otdf_python-0.2.8-py3-none-any.whl +pip install dist/otdf_python-0.2.9-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/build-scripts/uv_make_and_validate_script.sh b/build-scripts/uv_make_and_validate_script.sh index 2d2a374..e9f6ab6 100755 --- a/build-scripts/uv_make_and_validate_script.sh +++ b/build-scripts/uv_make_and_validate_script.sh @@ -70,7 +70,7 @@ loud_print "Installing wheel" uv venv .venv-wheel --python 3.12 "$PY_TYPE" source "${BUILD_ROOT}/.venv-wheel/bin/activate" pip install pybindgen -pip install dist/otdf_python-0.2.8-py3-none-any.whl +pip install dist/otdf_python-0.2.9-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/go.sum b/go.sum index 33b6f01..4d08aea 100644 --- a/go.sum +++ b/go.sum @@ -99,8 +99,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opentdf/platform/lib/fixtures v0.2.8 h1:lGYrMnbORtU62lxsJi8qPsxjFuNIkc4Dop8rVkH6pD0= -github.com/opentdf/platform/lib/fixtures v0.2.8/go.mod h1:8yCSe+oUzW9jbM573r9qgE68rjwDMNzktObiGVsO/W8= +github.com/opentdf/platform/lib/fixtures v0.2.9 h1:lGYrMnbORtU62lxsJi8qPsxjFuNIkc4Dop8rVkH6pD0= +github.com/opentdf/platform/lib/fixtures v0.2.9/go.mod h1:8yCSe+oUzW9jbM573r9qgE68rjwDMNzktObiGVsO/W8= github.com/opentdf/platform/lib/ocrypto v0.1.7 h1:IcCYRrwmMqntqUE8frmUDg5EZ0WMdldpGeGhbv9+/A8= github.com/opentdf/platform/lib/ocrypto v0.1.7/go.mod h1:4bhKPbRFzURMerH5Vr/LlszHvcoXQbfJXa0bpY7/7yg= github.com/opentdf/platform/protocol/go v0.2.22 h1:C/jjtwu5yTon8g0ewuN29QE7VXSQHyb2dx9W0U6Oqok= diff --git a/main.go b/main.go index c847dea..5010cc3 100644 --- a/main.go +++ b/main.go @@ -439,8 +439,8 @@ func EncryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O return nil, err } - var outputPaths []string - var errors []error + var outputPaths = make([]string, 0, len(files)) + var errors = make([]error, 0, len(files)) for _, file := range files { inputFilePath := file outputFilePath := inputFilePath + ".tdf" @@ -673,60 +673,86 @@ in the same directory as the input files, with the .tdf extension removed from t */ func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config OpentdfConfig) ([]string, error) { authScopes := []string{"email"} - sdkClient, err := newSdkClient(config, authScopes) - if err != nil { - return nil, err - } files, err := os.ReadDir(dirPath) if err != nil { return nil, err } - var outputPaths []string - var errors []error + outputPathsChan := make(chan string, len(files)) + errChan := make(chan error, len(files)) + + var wg sync.WaitGroup + for _, file := range files { if !file.IsDir() { for _, ext := range extensions { if strings.HasSuffix(file.Name(), ext) { - inputFilePath := filepath.Join(dirPath, file.Name()) - outputFilePath := strings.TrimSuffix(inputFilePath, ext) - - bytes, err := readBytesFromFile(inputFilePath) - if err != nil { - errors = append(errors, fmt.Errorf("failed to read file %s: %v", inputFilePath, err)) - continue - } - - decrypted, err := decryptBytesWithClient(bytes, sdkClient) - if err != nil { - errors = append(errors, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err)) - continue - } - - tdfFile, err := os.Create(outputFilePath) - if err != nil { - errors = append(errors, fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err)) - continue - } - defer tdfFile.Close() - - _, e := io.Copy(tdfFile, decrypted) - if e != nil { - errors = append(errors, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err)) - continue - } - - outputPaths = append(outputPaths, outputFilePath) + wg.Add(1) + go func(file os.DirEntry, ext string) { + defer wg.Done() + sdkClient, err := newSdkClient(config, authScopes) + if err != nil { + errChan <- fmt.Errorf("failed to create SDK client: %v", err) + return + } + + inputFilePath := filepath.Join(dirPath, file.Name()) + outputFilePath := strings.TrimSuffix(inputFilePath, ext) + + bytes, err := readBytesFromFile(inputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to read file %s: %v", inputFilePath, err) + return + } + + decrypted, err := decryptBytesWithClient(bytes, sdkClient) + if err != nil { + errChan <- fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err) + return + } + + tdfFile, err := os.Create(outputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err) + return + } + defer tdfFile.Close() + + _, e := io.Copy(tdfFile, decrypted) + if e != nil { + errChan <- fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err) + return + } + + outputPathsChan <- outputFilePath + }(file, ext) } } } } + wg.Wait() + close(outputPathsChan) + close(errChan) + + var outputPaths []string + for path := range outputPathsChan { + outputPaths = append(outputPaths, path) + } + + var errors []error + for err := range errChan { + errors = append(errors, err) + } + logOutputPaths(outputPaths, errors) - if len(errors) > 0 { - return outputPaths, fmt.Errorf("encountered errors during decryption: %v", errors) + if len(outputPaths) == 0 { + if len(errors) == 0 { + return nil, fmt.Errorf("no files with extensions %v found in directory %s", extensions, dirPath) + } + return nil, fmt.Errorf("encountered errors during decryption of files in directory %s: %v", dirPath, errors) } return outputPaths, nil } @@ -843,6 +869,7 @@ func findFiles(dir string, extensions []string) ([]string, error) { return files, nil } +// logOutputPaths logs the output paths and any errors that occurred during processing func logOutputPaths(outputPaths []string, errors []error) { if len(errors) > 0 { log.Println("Errors occurred during processing:") diff --git a/pyproject.toml b/pyproject.toml index 676d80f..3081133 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "otdf-python" # Should match 'setup.py' version number (used for gopy/pybindgen) -version = "0.2.8" +version = "0.2.9" description = "Unofficial OpenTDF SDK for Python." authors = [ {name="b-long", email="b-long@users.noreply.github.com"} @@ -19,7 +19,7 @@ pybindgen = "^0.22.1" [tool.poetry] package-mode = false -version = "0.2.8" +version = "0.2.9" [tool.poetry.dependencies] python = ">=3.11,<3.14" diff --git a/setup.py b/setup.py index 95dc039..a140562 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url="https://github.com/b-long/opentdf-python-sdk", package_data={"otdf_python": ["*.so"]}, # Should match 'pyproject.toml' version number - version="0.2.8", + version="0.2.9", author_email="b-long@users.noreply.github.com", include_package_data=True, ) diff --git a/setup_ci.py b/setup_ci.py index 5bb75af..bfac080 100644 --- a/setup_ci.py +++ b/setup_ci.py @@ -81,7 +81,7 @@ def build_extension(self, ext: Extension): setuptools.setup( name="otdf_python", - version="0.2.8", + version="0.2.9", author="b-long", description="Unofficial OpenTDF SDK for Python.", long_description_content_type="text/markdown", diff --git a/uv.lock b/uv.lock index 138c19b..59938cf 100644 --- a/uv.lock +++ b/uv.lock @@ -3,5 +3,5 @@ requires-python = ">=3.11" [[package]] name = "otdf-python" -version = "0.2.8" +version = "0.2.9" source = { editable = "." }