Use file extensions rather than glob patterns

.github/workflows/build-golang-macos.yaml

@@ -47,4 +47,4 @@ jobs:
 
       # - uses: ./.github/workflows/platform-integration-test.yaml
       #   with:
-      #     wheel: dist/otdf_python-0.2.6-py3-none-any.whl
+      #     wheel: dist/otdf_python-0.2.7-py3-none-any.whl

.github/workflows/build-golang-ubuntu.yaml

@@ -43,12 +43,12 @@ jobs:
 
       - uses: actions/cache/restore@v4
         with:
-          path: dist/otdf_python-0.2.6-py3-none-any.whl
+          path: dist/otdf_python-0.2.7-py3-none-any.whl
           key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }}
 
       - uses: actions/cache/save@v4
         with:
-          path: dist/otdf_python-0.2.6-py3-none-any.whl
+          path: dist/otdf_python-0.2.7-py3-none-any.whl
           key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}${{ matrix.python3_version }}-data-
@@ -61,5 +61,5 @@ jobs:
     needs: build
     uses: ./.github/workflows/platform-integration-test.yaml
     with:
-      wheel: dist/otdf_python-0.2.6-py3-none-any.whl
+      wheel: dist/otdf_python-0.2.7-py3-none-any.whl
       python_version: ${{ matrix.python3_version }}

.github/workflows/platform-integration-test.yaml

@@ -29,7 +29,7 @@ jobs:
 
       - uses: actions/cache/restore@v4
         with:
-          path: dist/otdf_python-0.2.6-py3-none-any.whl
+          path: dist/otdf_python-0.2.7-py3-none-any.whl
           key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }}
 
       - name: Prove that the input file is available

.pre-commit-config.yaml

@@ -33,7 +33,7 @@ repos:
           # - id: validate-toml
           - id: no-go-testing
     - repo: https://github.com/codespell-project/codespell
-      rev: v2.3.0
+      rev: v2.4.1
       hooks:
           - id: codespell
             args: ["--ignore-words-list", "b-long, otdf_python", "--skip=go.sum,otdf_python/"]

README.md

@@ -27,10 +27,10 @@ Install from the [Python Package Index (PyPI)](https://pypi.org):
 pip install otdf_python
 
 # Install a pinned version
-pip install otdf-python==0.2.6
+pip install otdf-python==0.2.7
 
 # Install a pinned version, from test.pypi.org
-pip install -i https://test.pypi.org/simple/ otdf-python==0.2.6
+pip install -i https://test.pypi.org/simple/ otdf-python==0.2.7
 ```
 
 ## Usage

build-scripts/ci-build.sh

@@ -72,4 +72,4 @@ echo "✨✨✨ Build wheel"
 poetry run python3 setup.py bdist_wheel
 
 echo "✨✨✨ Install wheel"
-pip install dist/otdf_python-0.2.6-py3-none-any.whl
+pip install dist/otdf_python-0.2.7-py3-none-any.whl

build-scripts/make_and_validate_script.sh

@@ -47,7 +47,7 @@ python3 -m pip install --upgrade setuptools wheel
 python3 setup.py bdist_wheel
 
 # Prove that the wheel can be installed
-pip install dist/otdf_python-0.2.6-py3-none-any.whl
+pip install dist/otdf_python-0.2.7-py3-none-any.whl
 
 if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then
     echo "Build is complete, skipping tests."

build-scripts/uv_make_and_validate_script.sh

@@ -70,7 +70,7 @@ loud_print "Installing wheel"
 uv venv .venv-wheel --python 3.12 "$PY_TYPE"
 source "${BUILD_ROOT}/.venv-wheel/bin/activate"
 pip install pybindgen
-pip install dist/otdf_python-0.2.6-py3-none-any.whl
+pip install dist/otdf_python-0.2.7-py3-none-any.whl
 
 if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then
     echo "Build is complete, skipping tests."

main.go

@@ -2,12 +2,17 @@ package gotdf_python
 
 /*
 All public (upper-case) functions here should be available to Python.
+* E.g. imported & tested via 'validate_otdf_python.py'
 
-As a result, all public functions should be imported & tested.
+TODO: Consider testing against attributes that are returned by some listing.
+* See: https://github.com/orgs/opentdf/discussions/947
 
-Currently, testing is performed via 'validate_otdf_python.py'
+TODO: Consider exposing an sdkClient that can be returned to the caller
+* Note, previously this failed in a 'gopy' compiled context
 
-FIXME: Expand test coverage, with known good attributes.  See: https://github.com/orgs/opentdf/discussions/947
+TODO: Platform knows about the IdP, perhaps we don't need to specify the TOKEN_ENDPOINT ?
+
+TODO: Expose authScopes []string (such as []string{"email"}) parameter to various functions
 
 */
 import (
@@ -404,33 +409,38 @@ func EncryptFilesInDirNPE(dirPath string, config OpentdfConfig, dataAttributes [
 }
 
 /*
-	EncryptFilesGlobNPE encrypts all files matching the specified glob pattern.
+	EncryptFilesWithExtensionsNPE encrypts all files in 'dirPath' with given file 'extensions'.
 
 Work is performed as an NPE (Non-Person Entity). Encrypted files are placed
 in the same directory as the input files, with a .tdf extension added to the file name.
 */
-func EncryptFilesGlobNPE(pattern string, config OpentdfConfig, dataAttributes []string) ([]string, error) {
+func EncryptFilesWithExtensionsNPE(dirPath string, extensions []string, config OpentdfConfig, dataAttributes []string) ([]string, error) {
 	authScopes := []string{"email"}
 	sdkClient, err := newSdkClient(config, authScopes)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err := filepath.Glob(pattern)
+	files, err := findFiles(dirPath, extensions)
 	if err != nil {
 		return nil, err
 	}
 
 	var outputPaths []string
-	for _, inputFilePath := range files {
+	var errors []error
+	for _, file := range files {
+		inputFilePath := file
 		outputFilePath := inputFilePath + ".tdf"
 		got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes)
 		if err != nil {
-
-			return nil, fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err)
-		} else {
-			outputPaths = append(outputPaths, got)
+			errors = append(errors, fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err))
+			continue
 		}
+		outputPaths = append(outputPaths, got)
+	}
+
+	if len(errors) > 0 {
+		return outputPaths, fmt.Errorf("encountered errors during encryption: %v", errors)
 	}
 	return outputPaths, nil
 }
@@ -483,17 +493,7 @@ func EncryptFilePE(inputFilePath string, outputFilePath string, config OpentdfCo
 }
 
 /*
-TODO: Create a single global var for sdkClinet (global var)
-  - E.g. in an HTTP server, create an instance for each connection
-
-TODO: The platform knows about the IdP, therefore we don't need
-to specify the TOKEN_ENDPOINT.
-
-TODO: Research why the platform is hard-coding "email" for scope
-
-A non-Public decrypt function.
-
-Based on:
+A non-Public decrypt function, based on:
 - https://github.com/opentdf/otdfctl/blob/46cfca1ba32c57f7264c320db27394c00412ca49/pkg/handlers/tdf.go#L29-L41
 */
 func decryptBytes(toDecrypt []byte, authScopes []string, config OpentdfConfig) (*bytes.Buffer, error) {
@@ -569,10 +569,6 @@ in the same directory as the input files, with the .tdf extension removed from t
 */
 func DecryptFilesInDirNPE(dirPath string, config OpentdfConfig) ([]string, error) {
 	authScopes := []string{"email"}
-	sdkClient, err := newSdkClient(config, authScopes)
-	if err != nil {
-		return nil, err
-	}
 
 	files, err := os.ReadDir(dirPath)
 	if err != nil {
@@ -588,6 +584,12 @@ func DecryptFilesInDirNPE(dirPath string, config OpentdfConfig) ([]string, error
 			wg.Add(1)
 			go func(file os.DirEntry) {
 				defer wg.Done()
+				sdkClient, err := newSdkClient(config, authScopes)
+				if err != nil {
+					errChan <- fmt.Errorf("failed to create SDK client: %v", err)
+					return
+				}
+
 				fileInfo, err := file.Info()
 				if err != nil {
 					errChan <- fmt.Errorf("failed to get file info for %s: %v", file.Name(), err)
@@ -643,53 +645,64 @@ func DecryptFilesInDirNPE(dirPath string, config OpentdfConfig) ([]string, error
 }
 
 /*
-DecryptFilesGlobNPE decrypts all files matching the specified glob pattern.
+DecryptFilesWithExtensionsNPE decrypts all files matching the file 'extensions' in 'dirPath'.
 Work is performed as an NPE (Non-Person Entity). Decrypted files are placed
 in the same directory as the input files, with the .tdf extension removed from the file name.
 */
-func DecryptFilesGlobNPE(pattern string, config OpentdfConfig) ([]string, error) {
+func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config OpentdfConfig) ([]string, error) {
 	authScopes := []string{"email"}
 	sdkClient, err := newSdkClient(config, authScopes)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err := filepath.Glob(pattern)
+	files, err := os.ReadDir(dirPath)
 	if err != nil {
 		return nil, err
 	}
 
 	var outputPaths []string
-	for _, inputFilePath := range files {
-		if strings.HasSuffix(inputFilePath, ".tdf") {
-			outputFilePath := strings.TrimSuffix(inputFilePath, ".tdf")
-
-			bytes, err := readBytesFromFile(inputFilePath)
-			if err != nil {
-				return nil, fmt.Errorf("failed to read file %s: %v", inputFilePath, err)
-			}
-
-			decrypted, err := decryptBytesWithClient(bytes, sdkClient)
-			if err != nil {
-				return nil, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err)
-			}
-
-			tdfFile, err := os.Create(outputFilePath)
-			if err != nil {
-				return nil, fmt.Errorf("failed to write decrypted file %s", outputFilePath)
-			}
-			defer tdfFile.Close()
-
-			_, e := io.Copy(tdfFile, decrypted)
-			if e != nil {
-				return nil, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err)
+	var errors []error
+	for _, file := range files {
+		if !file.IsDir() {
+			for _, ext := range extensions {
+				if strings.HasSuffix(file.Name(), ext) {
+					inputFilePath := filepath.Join(dirPath, file.Name())
+					outputFilePath := strings.TrimSuffix(inputFilePath, ext)
+
+					bytes, err := readBytesFromFile(inputFilePath)
+					if err != nil {
+						errors = append(errors, fmt.Errorf("failed to read file %s: %v", inputFilePath, err))
+						continue
+					}
+
+					decrypted, err := decryptBytesWithClient(bytes, sdkClient)
+					if err != nil {
+						errors = append(errors, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err))
+						continue
+					}
+
+					tdfFile, err := os.Create(outputFilePath)
+					if err != nil {
+						errors = append(errors, fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err))
+						continue
+					}
+					defer tdfFile.Close()
+
+					_, e := io.Copy(tdfFile, decrypted)
+					if e != nil {
+						errors = append(errors, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err))
+						continue
+					}
+
+					outputPaths = append(outputPaths, outputFilePath)
+				}
 			}
-
-			outputPaths = append(outputPaths, outputFilePath)
 		}
 	}
-	if len(outputPaths) == 0 {
-		return nil, fmt.Errorf("no files were decrypted")
+
+	if len(errors) > 0 {
+		return outputPaths, fmt.Errorf("encountered errors during decryption: %v", errors)
 	}
 	return outputPaths, nil
 }
@@ -779,3 +792,29 @@ func encryptBytesWithClient(b []byte, sdkClient *sdk.SDK, config OpentdfConfig,
 	}
 	return enc, nil
 }
+
+// Function to find all files recursively in a directory matching the given extensions
+func findFiles(dir string, extensions []string) ([]string, error) {
+	var files []string
+
+	// Use filepath.Walk to walk through the directory recursively
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			// If there's an error reading the file, skip it
+			return err
+		}
+
+		// Check if the file extension matches 'extensions' parameter
+		if !info.IsDir() && strings.Contains(strings.Join(extensions, ","), filepath.Ext(path)) {
+			files = append(files, path) // Add the file to the list
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return files, nil
+}