diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index 86910bc..f3f40b1 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -47,4 +47,4 @@ jobs: # - uses: ./.github/workflows/platform-integration-test.yaml # with: - # wheel: dist/otdf_python-0.2.4-py3-none-any.whl + # wheel: dist/otdf_python-0.2.5-py3-none-any.whl diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index b7b3122..3e18868 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -43,12 +43,12 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.4-py3-none-any.whl + path: dist/otdf_python-0.2.5-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} - uses: actions/cache/save@v4 with: - path: dist/otdf_python-0.2.4-py3-none-any.whl + path: dist/otdf_python-0.2.5-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} restore-keys: | ${{ runner.os }}${{ matrix.python3_version }}-data- @@ -61,5 +61,5 @@ jobs: needs: build uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.2.4-py3-none-any.whl + wheel: dist/otdf_python-0.2.5-py3-none-any.whl python_version: ${{ matrix.python3_version }} diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index b8619fb..1c6b168 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.4-py3-none-any.whl + path: dist/otdf_python-0.2.5-py3-none-any.whl key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }} - name: Prove that the input file is available diff --git a/README.md b/README.md index 74c5bfd..1d72d2d 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,10 @@ Install from the [Python Package Index (PyPI)](https://pypi.org): pip install otdf_python # Install a pinned version -pip install otdf-python==0.2.4 +pip install otdf-python==0.2.5 # Install a pinned version, from test.pypi.org -pip install -i https://test.pypi.org/simple/ otdf-python==0.2.4 +pip install -i https://test.pypi.org/simple/ otdf-python==0.2.5 ``` ## Usage diff --git a/build-scripts/ci-build.sh b/build-scripts/ci-build.sh index 6fc7090..b6dd391 100755 --- a/build-scripts/ci-build.sh +++ b/build-scripts/ci-build.sh @@ -72,4 +72,4 @@ echo "✨✨✨ Build wheel" poetry run python3 setup.py bdist_wheel echo "✨✨✨ Install wheel" -pip install dist/otdf_python-0.2.4-py3-none-any.whl +pip install dist/otdf_python-0.2.5-py3-none-any.whl diff --git a/build-scripts/make_and_validate_script.sh b/build-scripts/make_and_validate_script.sh index e6791cb..7df3762 100755 --- a/build-scripts/make_and_validate_script.sh +++ b/build-scripts/make_and_validate_script.sh @@ -47,7 +47,7 @@ python3 -m pip install --upgrade setuptools wheel python3 setup.py bdist_wheel # Prove that the wheel can be installed -pip install dist/otdf_python-0.2.4-py3-none-any.whl +pip install dist/otdf_python-0.2.5-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/build-scripts/uv_make_and_validate_script.sh b/build-scripts/uv_make_and_validate_script.sh index 34f11b4..37b1633 100755 --- a/build-scripts/uv_make_and_validate_script.sh +++ b/build-scripts/uv_make_and_validate_script.sh @@ -70,7 +70,7 @@ loud_print "Installing wheel" uv venv .venv-wheel --python 3.12 "$PY_TYPE" source "${BUILD_ROOT}/.venv-wheel/bin/activate" pip install pybindgen -pip install dist/otdf_python-0.2.4-py3-none-any.whl +pip install dist/otdf_python-0.2.5-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/main.go b/main.go index d8050ec..d06d26b 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ import ( "path" "path/filepath" "strings" + "sync" "github.com/opentdf/platform/sdk" ) @@ -374,18 +375,30 @@ func EncryptFilesInDirNPE(dirPath string, config OpentdfConfig, dataAttributes [ } var outputPaths []string + var mu sync.Mutex + var wg sync.WaitGroup + for _, file := range files { if !file.IsDir() { - inputFilePath := path.Join(dirPath, file.Name()) - outputFilePath := inputFilePath + ".tdf" - got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) - if err != nil { - return nil, fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err) - } else { + wg.Add(1) + go func(file os.DirEntry) { + defer wg.Done() + inputFilePath := path.Join(dirPath, file.Name()) + outputFilePath := inputFilePath + ".tdf" + got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) + if err != nil { + fmt.Printf("failed to encrypt file %s: %v\n", inputFilePath, err) + return + } + mu.Lock() outputPaths = append(outputPaths, got) - } + mu.Unlock() + }(file) } } + + wg.Wait() + return outputPaths, nil } @@ -565,37 +578,66 @@ func DecryptFilesInDirNPE(dirPath string, config OpentdfConfig) ([]string, error return nil, err } - var outputPaths []string + var wg sync.WaitGroup + outputPathsChan := make(chan string, len(files)) + errChan := make(chan error, len(files)) + for _, file := range files { if !file.IsDir() && strings.HasSuffix(file.Name(), ".tdf") { - inputFilePath := path.Join(dirPath, file.Name()) - outputFilePath := strings.TrimSuffix(inputFilePath, ".tdf") - - bytes, err := readBytesFromFile(inputFilePath) - if err != nil { - - return nil, fmt.Errorf("failed to read file %s: %v", inputFilePath, err) - } - - decrypted, err := decryptBytesWithClient(bytes, sdkClient) - if err != nil { - return nil, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err) - } + wg.Add(1) + go func(file os.DirEntry) { + defer wg.Done() + fileInfo, err := file.Info() + if err != nil { + errChan <- fmt.Errorf("failed to get file info for %s: %v", file.Name(), err) + return + } + inputFilePath := path.Join(dirPath, fileInfo.Name()) + outputFilePath := strings.TrimSuffix(inputFilePath, ".tdf") + + bytes, err := readBytesFromFile(inputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to read file %s: %v", inputFilePath, err) + return + } + + decrypted, err := decryptBytesWithClient(bytes, sdkClient) + if err != nil { + errChan <- fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err) + return + } + + tdfFile, err := os.Create(outputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err) + return + } + defer tdfFile.Close() + + _, e := io.Copy(tdfFile, decrypted) + if e != nil { + errChan <- fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err) + return + } + + outputPathsChan <- outputFilePath + }(file) + } + } - tdfFile, err := os.Create(outputFilePath) - if err != nil { - return nil, fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err) - } - defer tdfFile.Close() + wg.Wait() + close(outputPathsChan) + close(errChan) - _, e := io.Copy(tdfFile, decrypted) - if e != nil { - return nil, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err) - } + var outputPaths []string + for path := range outputPathsChan { + outputPaths = append(outputPaths, path) + } - outputPaths = append(outputPaths, outputFilePath) - } + if len(errChan) > 0 { + return nil, <-errChan } + return outputPaths, nil } diff --git a/pyproject.toml b/pyproject.toml index 77e5e7e..d8a98df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "otdf-python" # Should match 'setup.py' version number (used for gopy/pybindgen) -version = "0.2.4" +version = "0.2.5" description = "Unofficial OpenTDF SDK for Python." authors = [ {name="b-long", email="b-long@users.noreply.github.com"} @@ -19,7 +19,7 @@ pybindgen = "^0.22.1" [tool.poetry] package-mode = false -version = "0.2.4" +version = "0.2.5" [tool.poetry.dependencies] python = ">=3.11,<3.14" diff --git a/setup.py b/setup.py index ad39f39..eac1162 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url="https://github.com/b-long/opentdf-python-sdk", package_data={"otdf_python": ["*.so"]}, # Should match 'pyproject.toml' version number - version="0.2.4", + version="0.2.5", author_email="b-long@users.noreply.github.com", include_package_data=True, ) diff --git a/setup_ci.py b/setup_ci.py index 5dddaa2..e260e21 100644 --- a/setup_ci.py +++ b/setup_ci.py @@ -81,7 +81,7 @@ def build_extension(self, ext: Extension): setuptools.setup( name="otdf_python", - version="0.2.4", + version="0.2.5", author="b-long", description="Unofficial OpenTDF SDK for Python.", long_description_content_type="text/markdown", diff --git a/uv.lock b/uv.lock index 4c21e9b..55cc258 100644 --- a/uv.lock +++ b/uv.lock @@ -3,5 +3,5 @@ requires-python = ">=3.11" [[package]] name = "otdf-python" -version = "0.2.4" +version = "0.2.5" source = { editable = "." }