Use go routines for multi-file work

.github/workflows/build-golang-macos.yaml

@@ -47,4 +47,4 @@ jobs:
 
       # - uses: ./.github/workflows/platform-integration-test.yaml
       #   with:
-      #     wheel: dist/otdf_python-0.2.4-py3-none-any.whl
+      #     wheel: dist/otdf_python-0.2.5-py3-none-any.whl

.github/workflows/build-golang-ubuntu.yaml

@@ -43,12 +43,12 @@ jobs:
 
       - uses: actions/cache/restore@v4
         with:
-          path: dist/otdf_python-0.2.4-py3-none-any.whl
+          path: dist/otdf_python-0.2.5-py3-none-any.whl
           key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }}
 
       - uses: actions/cache/save@v4
         with:
-          path: dist/otdf_python-0.2.4-py3-none-any.whl
+          path: dist/otdf_python-0.2.5-py3-none-any.whl
           key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}${{ matrix.python3_version }}-data-
@@ -61,5 +61,5 @@ jobs:
     needs: build
     uses: ./.github/workflows/platform-integration-test.yaml
     with:
-      wheel: dist/otdf_python-0.2.4-py3-none-any.whl
+      wheel: dist/otdf_python-0.2.5-py3-none-any.whl
       python_version: ${{ matrix.python3_version }}

.github/workflows/platform-integration-test.yaml

@@ -29,7 +29,7 @@ jobs:
 
       - uses: actions/cache/restore@v4
         with:
-          path: dist/otdf_python-0.2.4-py3-none-any.whl
+          path: dist/otdf_python-0.2.5-py3-none-any.whl
           key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }}
 
       - name: Prove that the input file is available

README.md

@@ -27,10 +27,10 @@ Install from the [Python Package Index (PyPI)](https://pypi.org):
 pip install otdf_python
 
 # Install a pinned version
-pip install otdf-python==0.2.4
+pip install otdf-python==0.2.5
 
 # Install a pinned version, from test.pypi.org
-pip install -i https://test.pypi.org/simple/ otdf-python==0.2.4
+pip install -i https://test.pypi.org/simple/ otdf-python==0.2.5
 ```
 
 ## Usage

build-scripts/ci-build.sh

@@ -72,4 +72,4 @@ echo "✨✨✨ Build wheel"
 poetry run python3 setup.py bdist_wheel
 
 echo "✨✨✨ Install wheel"
-pip install dist/otdf_python-0.2.4-py3-none-any.whl
+pip install dist/otdf_python-0.2.5-py3-none-any.whl

build-scripts/make_and_validate_script.sh

@@ -47,7 +47,7 @@ python3 -m pip install --upgrade setuptools wheel
 python3 setup.py bdist_wheel
 
 # Prove that the wheel can be installed
-pip install dist/otdf_python-0.2.4-py3-none-any.whl
+pip install dist/otdf_python-0.2.5-py3-none-any.whl
 
 if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then
     echo "Build is complete, skipping tests."

build-scripts/uv_make_and_validate_script.sh

@@ -70,7 +70,7 @@ loud_print "Installing wheel"
 uv venv .venv-wheel --python 3.12 "$PY_TYPE"
 source "${BUILD_ROOT}/.venv-wheel/bin/activate"
 pip install pybindgen
-pip install dist/otdf_python-0.2.4-py3-none-any.whl
+pip install dist/otdf_python-0.2.5-py3-none-any.whl
 
 if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then
     echo "Build is complete, skipping tests."

main.go

@@ -20,6 +20,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/opentdf/platform/sdk"
 )
@@ -374,18 +375,30 @@ func EncryptFilesInDirNPE(dirPath string, config OpentdfConfig, dataAttributes [
 	}
 
 	var outputPaths []string
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+
 	for _, file := range files {
 		if !file.IsDir() {
-			inputFilePath := path.Join(dirPath, file.Name())
-			outputFilePath := inputFilePath + ".tdf"
-			got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes)
-			if err != nil {
-				return nil, fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err)
-			} else {
+			wg.Add(1)
+			go func(file os.DirEntry) {
+				defer wg.Done()
+				inputFilePath := path.Join(dirPath, file.Name())
+				outputFilePath := inputFilePath + ".tdf"
+				got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes)
+				if err != nil {
+					fmt.Printf("failed to encrypt file %s: %v\n", inputFilePath, err)
+					return
+				}
+				mu.Lock()
 				outputPaths = append(outputPaths, got)
-			}
+				mu.Unlock()
+			}(file)
 		}
 	}
+
+	wg.Wait()
+
 	return outputPaths, nil
 }
 
@@ -565,37 +578,66 @@ func DecryptFilesInDirNPE(dirPath string, config OpentdfConfig) ([]string, error
 		return nil, err
 	}
 
-	var outputPaths []string
+	var wg sync.WaitGroup
+	outputPathsChan := make(chan string, len(files))
+	errChan := make(chan error, len(files))
+
 	for _, file := range files {
 		if !file.IsDir() && strings.HasSuffix(file.Name(), ".tdf") {
-			inputFilePath := path.Join(dirPath, file.Name())
-			outputFilePath := strings.TrimSuffix(inputFilePath, ".tdf")
-
-			bytes, err := readBytesFromFile(inputFilePath)
-			if err != nil {
-
-				return nil, fmt.Errorf("failed to read file %s: %v", inputFilePath, err)
-			}
-
-			decrypted, err := decryptBytesWithClient(bytes, sdkClient)
-			if err != nil {
-				return nil, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err)
-			}
+			wg.Add(1)
+			go func(file os.DirEntry) {
+				defer wg.Done()
+				fileInfo, err := file.Info()
+				if err != nil {
+					errChan <- fmt.Errorf("failed to get file info for %s: %v", file.Name(), err)
+					return
+				}
+				inputFilePath := path.Join(dirPath, fileInfo.Name())
+				outputFilePath := strings.TrimSuffix(inputFilePath, ".tdf")
+
+				bytes, err := readBytesFromFile(inputFilePath)
+				if err != nil {
+					errChan <- fmt.Errorf("failed to read file %s: %v", inputFilePath, err)
+					return
+				}
+
+				decrypted, err := decryptBytesWithClient(bytes, sdkClient)
+				if err != nil {
+					errChan <- fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err)
+					return
+				}
+
+				tdfFile, err := os.Create(outputFilePath)
+				if err != nil {
+					errChan <- fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err)
+					return
+				}
+				defer tdfFile.Close()
+
+				_, e := io.Copy(tdfFile, decrypted)
+				if e != nil {
+					errChan <- fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err)
+					return
+				}
+
+				outputPathsChan <- outputFilePath
+			}(file)
+		}
+	}
 
-			tdfFile, err := os.Create(outputFilePath)
-			if err != nil {
-				return nil, fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err)
-			}
-			defer tdfFile.Close()
+	wg.Wait()
+	close(outputPathsChan)
+	close(errChan)
 
-			_, e := io.Copy(tdfFile, decrypted)
-			if e != nil {
-				return nil, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err)
-			}
+	var outputPaths []string
+	for path := range outputPathsChan {
+		outputPaths = append(outputPaths, path)
+	}
 
-			outputPaths = append(outputPaths, outputFilePath)
-		}
+	if len(errChan) > 0 {
+		return nil, <-errChan
 	}
+
 	return outputPaths, nil
 }
 

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "otdf-python"
 # Should match 'setup.py' version number (used for gopy/pybindgen)
-version = "0.2.4"
+version = "0.2.5"
 description = "Unofficial OpenTDF SDK for Python."
 authors = [
     {name="b-long", email="b-long@users.noreply.github.com"}
@@ -19,7 +19,7 @@ pybindgen = "^0.22.1"
 
 [tool.poetry]
 package-mode = false
-version = "0.2.4"
+version = "0.2.5"
 
 [tool.poetry.dependencies]
 python = ">=3.11,<3.14"

setup.py

@@ -24,7 +24,7 @@
     url="https://github.com/b-long/opentdf-python-sdk",
     package_data={"otdf_python": ["*.so"]},
     # Should match 'pyproject.toml' version number
-    version="0.2.4",
+    version="0.2.5",
     author_email="b-long@users.noreply.github.com",
     include_package_data=True,
 )

setup_ci.py

@@ -81,7 +81,7 @@ def build_extension(self, ext: Extension):
 
 setuptools.setup(
     name="otdf_python",
-    version="0.2.4",
+    version="0.2.5",
     author="b-long",
     description="Unofficial OpenTDF SDK for Python.",
     long_description_content_type="text/markdown",