Improve decrypt directory with extensions

.github/workflows/build-golang-macos.yaml

@@ -47,4 +47,4 @@ jobs:
 
       # - uses: ./.github/workflows/platform-integration-test.yaml
       #   with:
-      #     wheel: dist/otdf_python-0.2.8-py3-none-any.whl
+      #     wheel: dist/otdf_python-0.2.9-py3-none-any.whl

.github/workflows/build-golang-ubuntu.yaml

@@ -43,12 +43,12 @@ jobs:
 
       - uses: actions/cache/restore@v4
         with:
-          path: dist/otdf_python-0.2.8-py3-none-any.whl
+          path: dist/otdf_python-0.2.9-py3-none-any.whl
           key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }}
 
       - uses: actions/cache/save@v4
         with:
-          path: dist/otdf_python-0.2.8-py3-none-any.whl
+          path: dist/otdf_python-0.2.9-py3-none-any.whl
           key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}${{ matrix.python3_version }}-data-
@@ -61,5 +61,5 @@ jobs:
     needs: build
     uses: ./.github/workflows/platform-integration-test.yaml
     with:
-      wheel: dist/otdf_python-0.2.8-py3-none-any.whl
+      wheel: dist/otdf_python-0.2.9-py3-none-any.whl
       python_version: ${{ matrix.python3_version }}

.github/workflows/platform-integration-test.yaml

@@ -29,7 +29,7 @@ jobs:
 
       - uses: actions/cache/restore@v4
         with:
-          path: dist/otdf_python-0.2.8-py3-none-any.whl
+          path: dist/otdf_python-0.2.9-py3-none-any.whl
           key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }}
 
       - name: Prove that the input file is available

.github/workflows/publish-test.yaml

@@ -127,6 +127,7 @@ jobs:
   build_linux_arm:
     name: Linux Python ARM
     runs-on: ubuntu-22.04
+    timeout-minutes: 60
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/publish.yaml

@@ -127,6 +127,7 @@ jobs:
   build_linux_arm:
     name: Linux Python ARM
     runs-on: ubuntu-22.04
+    timeout-minutes: 60
 
     steps:
       - uses: actions/checkout@v4

README.md

@@ -27,10 +27,10 @@ Install from the [Python Package Index (PyPI)](https://pypi.org):
 pip install otdf_python
 
 # Install a pinned version
-pip install otdf-python==0.2.8
+pip install otdf-python==0.2.9
 
 # Install a pinned version, from test.pypi.org
-pip install -i https://test.pypi.org/simple/ otdf-python==0.2.8
+pip install -i https://test.pypi.org/simple/ otdf-python==0.2.9
 ```
 
 ## Usage

build-scripts/ci-build.sh

@@ -72,4 +72,4 @@ echo "✨✨✨ Build wheel"
 poetry run python3 setup.py bdist_wheel
 
 echo "✨✨✨ Install wheel"
-pip install dist/otdf_python-0.2.8-py3-none-any.whl
+pip install dist/otdf_python-0.2.9-py3-none-any.whl

build-scripts/make_and_validate_script.sh

@@ -47,7 +47,7 @@ python3 -m pip install --upgrade setuptools wheel
 python3 setup.py bdist_wheel
 
 # Prove that the wheel can be installed
-pip install dist/otdf_python-0.2.8-py3-none-any.whl
+pip install dist/otdf_python-0.2.9-py3-none-any.whl
 
 if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then
     echo "Build is complete, skipping tests."

build-scripts/uv_make_and_validate_script.sh

@@ -70,7 +70,7 @@ loud_print "Installing wheel"
 uv venv .venv-wheel --python 3.12 "$PY_TYPE"
 source "${BUILD_ROOT}/.venv-wheel/bin/activate"
 pip install pybindgen
-pip install dist/otdf_python-0.2.8-py3-none-any.whl
+pip install dist/otdf_python-0.2.9-py3-none-any.whl
 
 if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then
     echo "Build is complete, skipping tests."

go.sum

@@ -99,8 +99,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
 github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
-github.com/opentdf/platform/lib/fixtures v0.2.8 h1:lGYrMnbORtU62lxsJi8qPsxjFuNIkc4Dop8rVkH6pD0=
-github.com/opentdf/platform/lib/fixtures v0.2.8/go.mod h1:8yCSe+oUzW9jbM573r9qgE68rjwDMNzktObiGVsO/W8=
+github.com/opentdf/platform/lib/fixtures v0.2.9 h1:lGYrMnbORtU62lxsJi8qPsxjFuNIkc4Dop8rVkH6pD0=
+github.com/opentdf/platform/lib/fixtures v0.2.9/go.mod h1:8yCSe+oUzW9jbM573r9qgE68rjwDMNzktObiGVsO/W8=
 github.com/opentdf/platform/lib/ocrypto v0.1.7 h1:IcCYRrwmMqntqUE8frmUDg5EZ0WMdldpGeGhbv9+/A8=
 github.com/opentdf/platform/lib/ocrypto v0.1.7/go.mod h1:4bhKPbRFzURMerH5Vr/LlszHvcoXQbfJXa0bpY7/7yg=
 github.com/opentdf/platform/protocol/go v0.2.22 h1:C/jjtwu5yTon8g0ewuN29QE7VXSQHyb2dx9W0U6Oqok=

main.go

@@ -439,8 +439,8 @@ func EncryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O
 		return nil, err
 	}
 
-	var outputPaths []string
-	var errors []error
+	var outputPaths = make([]string, 0, len(files))
+	var errors = make([]error, 0, len(files))
 	for _, file := range files {
 		inputFilePath := file
 		outputFilePath := inputFilePath + ".tdf"
@@ -673,60 +673,86 @@ in the same directory as the input files, with the .tdf extension removed from t
 */
 func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config OpentdfConfig) ([]string, error) {
 	authScopes := []string{"email"}
-	sdkClient, err := newSdkClient(config, authScopes)
-	if err != nil {
-		return nil, err
-	}
 
 	files, err := os.ReadDir(dirPath)
 	if err != nil {
 		return nil, err
 	}
 
-	var outputPaths []string
-	var errors []error
+	outputPathsChan := make(chan string, len(files))
+	errChan := make(chan error, len(files))
+
+	var wg sync.WaitGroup
+
 	for _, file := range files {
 		if !file.IsDir() {
 			for _, ext := range extensions {
 				if strings.HasSuffix(file.Name(), ext) {
-					inputFilePath := filepath.Join(dirPath, file.Name())
-					outputFilePath := strings.TrimSuffix(inputFilePath, ext)
-
-					bytes, err := readBytesFromFile(inputFilePath)
-					if err != nil {
-						errors = append(errors, fmt.Errorf("failed to read file %s: %v", inputFilePath, err))
-						continue
-					}
-
-					decrypted, err := decryptBytesWithClient(bytes, sdkClient)
-					if err != nil {
-						errors = append(errors, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err))
-						continue
-					}
-
-					tdfFile, err := os.Create(outputFilePath)
-					if err != nil {
-						errors = append(errors, fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err))
-						continue
-					}
-					defer tdfFile.Close()
-
-					_, e := io.Copy(tdfFile, decrypted)
-					if e != nil {
-						errors = append(errors, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err))
-						continue
-					}
-
-					outputPaths = append(outputPaths, outputFilePath)
+					wg.Add(1)
+					go func(file os.DirEntry, ext string) {
+						defer wg.Done()
+						sdkClient, err := newSdkClient(config, authScopes)
+						if err != nil {
+							errChan <- fmt.Errorf("failed to create SDK client: %v", err)
+							return
+						}
+
+						inputFilePath := filepath.Join(dirPath, file.Name())
+						outputFilePath := strings.TrimSuffix(inputFilePath, ext)
+
+						bytes, err := readBytesFromFile(inputFilePath)
+						if err != nil {
+							errChan <- fmt.Errorf("failed to read file %s: %v", inputFilePath, err)
+							return
+						}
+
+						decrypted, err := decryptBytesWithClient(bytes, sdkClient)
+						if err != nil {
+							errChan <- fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err)
+							return
+						}
+
+						tdfFile, err := os.Create(outputFilePath)
+						if err != nil {
+							errChan <- fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err)
+							return
+						}
+						defer tdfFile.Close()
+
+						_, e := io.Copy(tdfFile, decrypted)
+						if e != nil {
+							errChan <- fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err)
+							return
+						}
+
+						outputPathsChan <- outputFilePath
+					}(file, ext)
 				}
 			}
 		}
 	}
 
+	wg.Wait()
+	close(outputPathsChan)
+	close(errChan)
+
+	var outputPaths []string
+	for path := range outputPathsChan {
+		outputPaths = append(outputPaths, path)
+	}
+
+	var errors []error
+	for err := range errChan {
+		errors = append(errors, err)
+	}
+
 	logOutputPaths(outputPaths, errors)
 
-	if len(errors) > 0 {
-		return outputPaths, fmt.Errorf("encountered errors during decryption: %v", errors)
+	if len(outputPaths) == 0 {
+		if len(errors) == 0 {
+			return nil, fmt.Errorf("no files with extensions %v found in directory %s", extensions, dirPath)
+		}
+		return nil, fmt.Errorf("encountered errors during decryption of files in directory %s: %v", dirPath, errors)
 	}
 	return outputPaths, nil
 }
@@ -843,6 +869,7 @@ func findFiles(dir string, extensions []string) ([]string, error) {
 	return files, nil
 }
 
+// logOutputPaths logs the output paths and any errors that occurred during processing
 func logOutputPaths(outputPaths []string, errors []error) {
 	if len(errors) > 0 {
 		log.Println("Errors occurred during processing:")

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "otdf-python"
 # Should match 'setup.py' version number (used for gopy/pybindgen)
-version = "0.2.8"
+version = "0.2.9"
 description = "Unofficial OpenTDF SDK for Python."
 authors = [
     {name="b-long", email="b-long@users.noreply.github.com"}
@@ -19,7 +19,7 @@ pybindgen = "^0.22.1"
 
 [tool.poetry]
 package-mode = false
-version = "0.2.8"
+version = "0.2.9"
 
 [tool.poetry.dependencies]
 python = ">=3.11,<3.14"

setup.py

@@ -24,7 +24,7 @@
     url="https://github.com/b-long/opentdf-python-sdk",
     package_data={"otdf_python": ["*.so"]},
     # Should match 'pyproject.toml' version number
-    version="0.2.8",
+    version="0.2.9",
     author_email="b-long@users.noreply.github.com",
     include_package_data=True,
 )

setup_ci.py

@@ -81,7 +81,7 @@ def build_extension(self, ext: Extension):
 
 setuptools.setup(
     name="otdf_python",
-    version="0.2.8",
+    version="0.2.9",
     author="b-long",
     description="Unofficial OpenTDF SDK for Python.",
     long_description_content_type="text/markdown",