From fa121dd715b47c8badac6bf244e9cfc2669d2f01 Mon Sep 17 00:00:00 2001 From: b-long Date: Thu, 6 Feb 2025 15:39:41 -0500 Subject: [PATCH 01/19] Use go routines for multifile work --- main.go | 106 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 32 deletions(-) diff --git a/main.go b/main.go index d8050ec..d06d26b 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ import ( "path" "path/filepath" "strings" + "sync" "github.com/opentdf/platform/sdk" ) @@ -374,18 +375,30 @@ func EncryptFilesInDirNPE(dirPath string, config OpentdfConfig, dataAttributes [ } var outputPaths []string + var mu sync.Mutex + var wg sync.WaitGroup + for _, file := range files { if !file.IsDir() { - inputFilePath := path.Join(dirPath, file.Name()) - outputFilePath := inputFilePath + ".tdf" - got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) - if err != nil { - return nil, fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err) - } else { + wg.Add(1) + go func(file os.DirEntry) { + defer wg.Done() + inputFilePath := path.Join(dirPath, file.Name()) + outputFilePath := inputFilePath + ".tdf" + got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) + if err != nil { + fmt.Printf("failed to encrypt file %s: %v\n", inputFilePath, err) + return + } + mu.Lock() outputPaths = append(outputPaths, got) - } + mu.Unlock() + }(file) } } + + wg.Wait() + return outputPaths, nil } @@ -565,37 +578,66 @@ func DecryptFilesInDirNPE(dirPath string, config OpentdfConfig) ([]string, error return nil, err } - var outputPaths []string + var wg sync.WaitGroup + outputPathsChan := make(chan string, len(files)) + errChan := make(chan error, len(files)) + for _, file := range files { if !file.IsDir() && strings.HasSuffix(file.Name(), ".tdf") { - inputFilePath := path.Join(dirPath, file.Name()) - outputFilePath := strings.TrimSuffix(inputFilePath, ".tdf") - - bytes, err := readBytesFromFile(inputFilePath) - if err != nil { - - return nil, fmt.Errorf("failed to read file %s: %v", inputFilePath, err) - } - - decrypted, err := decryptBytesWithClient(bytes, sdkClient) - if err != nil { - return nil, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err) - } + wg.Add(1) + go func(file os.DirEntry) { + defer wg.Done() + fileInfo, err := file.Info() + if err != nil { + errChan <- fmt.Errorf("failed to get file info for %s: %v", file.Name(), err) + return + } + inputFilePath := path.Join(dirPath, fileInfo.Name()) + outputFilePath := strings.TrimSuffix(inputFilePath, ".tdf") + + bytes, err := readBytesFromFile(inputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to read file %s: %v", inputFilePath, err) + return + } + + decrypted, err := decryptBytesWithClient(bytes, sdkClient) + if err != nil { + errChan <- fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err) + return + } + + tdfFile, err := os.Create(outputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err) + return + } + defer tdfFile.Close() + + _, e := io.Copy(tdfFile, decrypted) + if e != nil { + errChan <- fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err) + return + } + + outputPathsChan <- outputFilePath + }(file) + } + } - tdfFile, err := os.Create(outputFilePath) - if err != nil { - return nil, fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err) - } - defer tdfFile.Close() + wg.Wait() + close(outputPathsChan) + close(errChan) - _, e := io.Copy(tdfFile, decrypted) - if e != nil { - return nil, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err) - } + var outputPaths []string + for path := range outputPathsChan { + outputPaths = append(outputPaths, path) + } - outputPaths = append(outputPaths, outputFilePath) - } + if len(errChan) > 0 { + return nil, <-errChan } + return outputPaths, nil } From ca5ad9c39f795be270653f2237c4d40c0a4f7d37 Mon Sep 17 00:00:00 2001 From: b-long Date: Thu, 6 Feb 2025 15:59:02 -0500 Subject: [PATCH 02/19] Bump 0.2.4 -> 0.2.5 --- .github/workflows/build-golang-macos.yaml | 2 +- .github/workflows/build-golang-ubuntu.yaml | 6 +++--- .github/workflows/platform-integration-test.yaml | 2 +- README.md | 4 ++-- build-scripts/ci-build.sh | 2 +- build-scripts/make_and_validate_script.sh | 2 +- build-scripts/uv_make_and_validate_script.sh | 2 +- pyproject.toml | 4 ++-- setup.py | 2 +- setup_ci.py | 2 +- uv.lock | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index 86910bc..f3f40b1 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -47,4 +47,4 @@ jobs: # - uses: ./.github/workflows/platform-integration-test.yaml # with: - # wheel: dist/otdf_python-0.2.4-py3-none-any.whl + # wheel: dist/otdf_python-0.2.5-py3-none-any.whl diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index b7b3122..3e18868 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -43,12 +43,12 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.4-py3-none-any.whl + path: dist/otdf_python-0.2.5-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} - uses: actions/cache/save@v4 with: - path: dist/otdf_python-0.2.4-py3-none-any.whl + path: dist/otdf_python-0.2.5-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} restore-keys: | ${{ runner.os }}${{ matrix.python3_version }}-data- @@ -61,5 +61,5 @@ jobs: needs: build uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.2.4-py3-none-any.whl + wheel: dist/otdf_python-0.2.5-py3-none-any.whl python_version: ${{ matrix.python3_version }} diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index b8619fb..1c6b168 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.4-py3-none-any.whl + path: dist/otdf_python-0.2.5-py3-none-any.whl key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }} - name: Prove that the input file is available diff --git a/README.md b/README.md index 74c5bfd..1d72d2d 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,10 @@ Install from the [Python Package Index (PyPI)](https://pypi.org): pip install otdf_python # Install a pinned version -pip install otdf-python==0.2.4 +pip install otdf-python==0.2.5 # Install a pinned version, from test.pypi.org -pip install -i https://test.pypi.org/simple/ otdf-python==0.2.4 +pip install -i https://test.pypi.org/simple/ otdf-python==0.2.5 ``` ## Usage diff --git a/build-scripts/ci-build.sh b/build-scripts/ci-build.sh index 6fc7090..b6dd391 100755 --- a/build-scripts/ci-build.sh +++ b/build-scripts/ci-build.sh @@ -72,4 +72,4 @@ echo "✨✨✨ Build wheel" poetry run python3 setup.py bdist_wheel echo "✨✨✨ Install wheel" -pip install dist/otdf_python-0.2.4-py3-none-any.whl +pip install dist/otdf_python-0.2.5-py3-none-any.whl diff --git a/build-scripts/make_and_validate_script.sh b/build-scripts/make_and_validate_script.sh index e6791cb..7df3762 100755 --- a/build-scripts/make_and_validate_script.sh +++ b/build-scripts/make_and_validate_script.sh @@ -47,7 +47,7 @@ python3 -m pip install --upgrade setuptools wheel python3 setup.py bdist_wheel # Prove that the wheel can be installed -pip install dist/otdf_python-0.2.4-py3-none-any.whl +pip install dist/otdf_python-0.2.5-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/build-scripts/uv_make_and_validate_script.sh b/build-scripts/uv_make_and_validate_script.sh index 34f11b4..37b1633 100755 --- a/build-scripts/uv_make_and_validate_script.sh +++ b/build-scripts/uv_make_and_validate_script.sh @@ -70,7 +70,7 @@ loud_print "Installing wheel" uv venv .venv-wheel --python 3.12 "$PY_TYPE" source "${BUILD_ROOT}/.venv-wheel/bin/activate" pip install pybindgen -pip install dist/otdf_python-0.2.4-py3-none-any.whl +pip install dist/otdf_python-0.2.5-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/pyproject.toml b/pyproject.toml index 77e5e7e..d8a98df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "otdf-python" # Should match 'setup.py' version number (used for gopy/pybindgen) -version = "0.2.4" +version = "0.2.5" description = "Unofficial OpenTDF SDK for Python." authors = [ {name="b-long", email="b-long@users.noreply.github.com"} @@ -19,7 +19,7 @@ pybindgen = "^0.22.1" [tool.poetry] package-mode = false -version = "0.2.4" +version = "0.2.5" [tool.poetry.dependencies] python = ">=3.11,<3.14" diff --git a/setup.py b/setup.py index ad39f39..eac1162 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url="https://github.com/b-long/opentdf-python-sdk", package_data={"otdf_python": ["*.so"]}, # Should match 'pyproject.toml' version number - version="0.2.4", + version="0.2.5", author_email="b-long@users.noreply.github.com", include_package_data=True, ) diff --git a/setup_ci.py b/setup_ci.py index 5dddaa2..e260e21 100644 --- a/setup_ci.py +++ b/setup_ci.py @@ -81,7 +81,7 @@ def build_extension(self, ext: Extension): setuptools.setup( name="otdf_python", - version="0.2.4", + version="0.2.5", author="b-long", description="Unofficial OpenTDF SDK for Python.", long_description_content_type="text/markdown", diff --git a/uv.lock b/uv.lock index 4c21e9b..55cc258 100644 --- a/uv.lock +++ b/uv.lock @@ -3,5 +3,5 @@ requires-python = ">=3.11" [[package]] name = "otdf-python" -version = "0.2.4" +version = "0.2.5" source = { editable = "." } From 0e4f84a4f0156a4d1cfce333c5a102e1e3d105b1 Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 7 Feb 2025 10:20:33 -0500 Subject: [PATCH 03/19] Use one client per go routine --- main.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index d06d26b..dc27a5e 100644 --- a/main.go +++ b/main.go @@ -364,10 +364,6 @@ in the same directory as the input files, with a .tdf extension added to the fil */ func EncryptFilesInDirNPE(dirPath string, config OpentdfConfig, dataAttributes []string) ([]string, error) { authScopes := []string{"email"} - sdkClient, err := newSdkClient(config, authScopes) - if err != nil { - return nil, err - } files, err := os.ReadDir(dirPath) if err != nil { @@ -383,6 +379,11 @@ func EncryptFilesInDirNPE(dirPath string, config OpentdfConfig, dataAttributes [ wg.Add(1) go func(file os.DirEntry) { defer wg.Done() + sdkClient, err := newSdkClient(config, authScopes) + if err != nil { + fmt.Printf("failed to create SDK client: %v\n", err) + return + } inputFilePath := path.Join(dirPath, file.Name()) outputFilePath := inputFilePath + ".tdf" got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) From 2ed949bf9cbf055c5898abe4a64ef8f541bed470 Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 7 Feb 2025 10:32:03 -0500 Subject: [PATCH 04/19] Bump 0.2.5 -> 0.2.6 --- .github/workflows/build-golang-macos.yaml | 2 +- .github/workflows/build-golang-ubuntu.yaml | 6 +++--- .github/workflows/platform-integration-test.yaml | 2 +- README.md | 4 ++-- build-scripts/ci-build.sh | 2 +- build-scripts/make_and_validate_script.sh | 2 +- build-scripts/uv_make_and_validate_script.sh | 2 +- pyproject.toml | 4 ++-- setup.py | 2 +- setup_ci.py | 2 +- uv.lock | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index f3f40b1..d1d165f 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -47,4 +47,4 @@ jobs: # - uses: ./.github/workflows/platform-integration-test.yaml # with: - # wheel: dist/otdf_python-0.2.5-py3-none-any.whl + # wheel: dist/otdf_python-0.2.6-py3-none-any.whl diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index 3e18868..a98b148 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -43,12 +43,12 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.5-py3-none-any.whl + path: dist/otdf_python-0.2.6-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} - uses: actions/cache/save@v4 with: - path: dist/otdf_python-0.2.5-py3-none-any.whl + path: dist/otdf_python-0.2.6-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} restore-keys: | ${{ runner.os }}${{ matrix.python3_version }}-data- @@ -61,5 +61,5 @@ jobs: needs: build uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.2.5-py3-none-any.whl + wheel: dist/otdf_python-0.2.6-py3-none-any.whl python_version: ${{ matrix.python3_version }} diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index 1c6b168..e80a861 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.5-py3-none-any.whl + path: dist/otdf_python-0.2.6-py3-none-any.whl key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }} - name: Prove that the input file is available diff --git a/README.md b/README.md index 1d72d2d..ae9752b 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,10 @@ Install from the [Python Package Index (PyPI)](https://pypi.org): pip install otdf_python # Install a pinned version -pip install otdf-python==0.2.5 +pip install otdf-python==0.2.6 # Install a pinned version, from test.pypi.org -pip install -i https://test.pypi.org/simple/ otdf-python==0.2.5 +pip install -i https://test.pypi.org/simple/ otdf-python==0.2.6 ``` ## Usage diff --git a/build-scripts/ci-build.sh b/build-scripts/ci-build.sh index b6dd391..394e377 100755 --- a/build-scripts/ci-build.sh +++ b/build-scripts/ci-build.sh @@ -72,4 +72,4 @@ echo "✨✨✨ Build wheel" poetry run python3 setup.py bdist_wheel echo "✨✨✨ Install wheel" -pip install dist/otdf_python-0.2.5-py3-none-any.whl +pip install dist/otdf_python-0.2.6-py3-none-any.whl diff --git a/build-scripts/make_and_validate_script.sh b/build-scripts/make_and_validate_script.sh index 7df3762..b151ccd 100755 --- a/build-scripts/make_and_validate_script.sh +++ b/build-scripts/make_and_validate_script.sh @@ -47,7 +47,7 @@ python3 -m pip install --upgrade setuptools wheel python3 setup.py bdist_wheel # Prove that the wheel can be installed -pip install dist/otdf_python-0.2.5-py3-none-any.whl +pip install dist/otdf_python-0.2.6-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/build-scripts/uv_make_and_validate_script.sh b/build-scripts/uv_make_and_validate_script.sh index 37b1633..d9bef3b 100755 --- a/build-scripts/uv_make_and_validate_script.sh +++ b/build-scripts/uv_make_and_validate_script.sh @@ -70,7 +70,7 @@ loud_print "Installing wheel" uv venv .venv-wheel --python 3.12 "$PY_TYPE" source "${BUILD_ROOT}/.venv-wheel/bin/activate" pip install pybindgen -pip install dist/otdf_python-0.2.5-py3-none-any.whl +pip install dist/otdf_python-0.2.6-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/pyproject.toml b/pyproject.toml index d8a98df..7ab6610 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "otdf-python" # Should match 'setup.py' version number (used for gopy/pybindgen) -version = "0.2.5" +version = "0.2.6" description = "Unofficial OpenTDF SDK for Python." authors = [ {name="b-long", email="b-long@users.noreply.github.com"} @@ -19,7 +19,7 @@ pybindgen = "^0.22.1" [tool.poetry] package-mode = false -version = "0.2.5" +version = "0.2.6" [tool.poetry.dependencies] python = ">=3.11,<3.14" diff --git a/setup.py b/setup.py index eac1162..c270834 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url="https://github.com/b-long/opentdf-python-sdk", package_data={"otdf_python": ["*.so"]}, # Should match 'pyproject.toml' version number - version="0.2.5", + version="0.2.6", author_email="b-long@users.noreply.github.com", include_package_data=True, ) diff --git a/setup_ci.py b/setup_ci.py index e260e21..a602293 100644 --- a/setup_ci.py +++ b/setup_ci.py @@ -81,7 +81,7 @@ def build_extension(self, ext: Extension): setuptools.setup( name="otdf_python", - version="0.2.5", + version="0.2.6", author="b-long", description="Unofficial OpenTDF SDK for Python.", long_description_content_type="text/markdown", diff --git a/uv.lock b/uv.lock index 55cc258..72e576a 100644 --- a/uv.lock +++ b/uv.lock @@ -3,5 +3,5 @@ requires-python = ">=3.11" [[package]] name = "otdf-python" -version = "0.2.5" +version = "0.2.6" source = { editable = "." } From 1f3c04828204fe09cec71716733cf3d24fba6178 Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 7 Feb 2025 16:12:10 -0500 Subject: [PATCH 05/19] Use file extensions not glob pattern --- main.go | 127 +++++++++++++++++++++++++------------------- otdf_python_test.go | 70 ++++++++++++++++-------- 2 files changed, 120 insertions(+), 77 deletions(-) diff --git a/main.go b/main.go index dc27a5e..5d435ab 100644 --- a/main.go +++ b/main.go @@ -2,12 +2,17 @@ package gotdf_python /* All public (upper-case) functions here should be available to Python. +* E.g. imported & tested via 'validate_otdf_python.py' -As a result, all public functions should be imported & tested. +TODO: Consider testing against attributes that are returned by some listing. +* See: https://github.com/orgs/opentdf/discussions/947 -Currently, testing is performed via 'validate_otdf_python.py' +TODO: Consider exposing an sdkClient that can be returned to the caller +* Note, previously this failed in a 'gopy' compiled context -FIXME: Expand test coverage, with known good attributes. See: https://github.com/orgs/opentdf/discussions/947 +TODO: Platform knows about the IdP, perhaps we don't need to specify the TOKEN_ENDPOINT ? + +TODO: Expose authScopes []string (such as []string{"email"}) parameter to various functions */ import ( @@ -404,34 +409,45 @@ func EncryptFilesInDirNPE(dirPath string, config OpentdfConfig, dataAttributes [ } /* - EncryptFilesGlobNPE encrypts all files matching the specified glob pattern. + EncryptFilesWithExtensionsNPE encrypts all files in 'dirPath' with given file 'extensions'. Work is performed as an NPE (Non-Person Entity). Encrypted files are placed in the same directory as the input files, with a .tdf extension added to the file name. */ -func EncryptFilesGlobNPE(pattern string, config OpentdfConfig, dataAttributes []string) ([]string, error) { +func EncryptFilesWithExtensionsNPE(dirPath string, extensions []string, config OpentdfConfig, dataAttributes []string) ([]string, error) { authScopes := []string{"email"} sdkClient, err := newSdkClient(config, authScopes) if err != nil { return nil, err } - files, err := filepath.Glob(pattern) + files, err := os.ReadDir(dirPath) if err != nil { return nil, err } var outputPaths []string - for _, inputFilePath := range files { - outputFilePath := inputFilePath + ".tdf" - got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) - if err != nil { - - return nil, fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err) - } else { - outputPaths = append(outputPaths, got) + var errors []error + for _, file := range files { + if !file.IsDir() { + for _, ext := range extensions { + if strings.HasSuffix(file.Name(), ext) { + inputFilePath := filepath.Join(dirPath, file.Name()) + outputFilePath := inputFilePath + ".tdf" + got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) + if err != nil { + errors = append(errors, fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err)) + continue + } + outputPaths = append(outputPaths, got) + } + } } } + + if len(errors) > 0 { + return outputPaths, fmt.Errorf("encountered errors during encryption: %v", errors) + } return outputPaths, nil } @@ -483,17 +499,7 @@ func EncryptFilePE(inputFilePath string, outputFilePath string, config OpentdfCo } /* -TODO: Create a single global var for sdkClinet (global var) - - E.g. in an HTTP server, create an instance for each connection - -TODO: The platform knows about the IdP, therefore we don't need -to specify the TOKEN_ENDPOINT. - -TODO: Research why the platform is hard-coding "email" for scope - -A non-Public decrypt function. - -Based on: +A non-Public decrypt function, based on: - https://github.com/opentdf/otdfctl/blob/46cfca1ba32c57f7264c320db27394c00412ca49/pkg/handlers/tdf.go#L29-L41 */ func decryptBytes(toDecrypt []byte, authScopes []string, config OpentdfConfig) (*bytes.Buffer, error) { @@ -643,53 +649,64 @@ func DecryptFilesInDirNPE(dirPath string, config OpentdfConfig) ([]string, error } /* -DecryptFilesGlobNPE decrypts all files matching the specified glob pattern. +DecryptFilesWithExtensionsNPE decrypts all files matching the file 'extensions' in 'dirPath'. Work is performed as an NPE (Non-Person Entity). Decrypted files are placed in the same directory as the input files, with the .tdf extension removed from the file name. */ -func DecryptFilesGlobNPE(pattern string, config OpentdfConfig) ([]string, error) { +func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config OpentdfConfig) ([]string, error) { authScopes := []string{"email"} sdkClient, err := newSdkClient(config, authScopes) if err != nil { return nil, err } - files, err := filepath.Glob(pattern) + files, err := os.ReadDir(dirPath) if err != nil { return nil, err } var outputPaths []string - for _, inputFilePath := range files { - if strings.HasSuffix(inputFilePath, ".tdf") { - outputFilePath := strings.TrimSuffix(inputFilePath, ".tdf") - - bytes, err := readBytesFromFile(inputFilePath) - if err != nil { - return nil, fmt.Errorf("failed to read file %s: %v", inputFilePath, err) - } - - decrypted, err := decryptBytesWithClient(bytes, sdkClient) - if err != nil { - return nil, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err) - } - - tdfFile, err := os.Create(outputFilePath) - if err != nil { - return nil, fmt.Errorf("failed to write decrypted file %s", outputFilePath) - } - defer tdfFile.Close() - - _, e := io.Copy(tdfFile, decrypted) - if e != nil { - return nil, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err) + var errors []error + for _, file := range files { + if !file.IsDir() { + for _, ext := range extensions { + if strings.HasSuffix(file.Name(), ext) { + inputFilePath := filepath.Join(dirPath, file.Name()) + outputFilePath := strings.TrimSuffix(inputFilePath, ext) + + bytes, err := readBytesFromFile(inputFilePath) + if err != nil { + errors = append(errors, fmt.Errorf("failed to read file %s: %v", inputFilePath, err)) + continue + } + + decrypted, err := decryptBytesWithClient(bytes, sdkClient) + if err != nil { + errors = append(errors, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err)) + continue + } + + tdfFile, err := os.Create(outputFilePath) + if err != nil { + errors = append(errors, fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err)) + continue + } + defer tdfFile.Close() + + _, e := io.Copy(tdfFile, decrypted) + if e != nil { + errors = append(errors, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err)) + continue + } + + outputPaths = append(outputPaths, outputFilePath) + } } - - outputPaths = append(outputPaths, outputFilePath) } } - if len(outputPaths) == 0 { - return nil, fmt.Errorf("no files were decrypted") + + if len(errors) > 0 { + return outputPaths, fmt.Errorf("encountered errors during decryption: %v", errors) } return outputPaths, nil } diff --git a/otdf_python_test.go b/otdf_python_test.go index 939e940..95db6d0 100644 --- a/otdf_python_test.go +++ b/otdf_python_test.go @@ -378,26 +378,40 @@ func Test_NPE_Encrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { t.Fatal("Unable to write to temporary file", err) } - // Call the EncryptFilesInDirNPE function - got, err := gotdf_python.EncryptFilesInDirNPE(tmpDir, gotdf_python.OpentdfConfig{ + // Create a temporary file in the directory + tmpFile3, err := os.CreateTemp(tmpDir, "input-file3-*.csv") + if err != nil { + t.Fatal("Could not create input file", err) + } + defer tmpFile3.Close() + + // Write some data to the file + if _, err = tmpFile3.WriteString("test data"); err != nil { + t.Fatal("Unable to write to temporary file", err) + } + + cfg := gotdf_python.OpentdfConfig{ ClientId: config.npeClientId, ClientSecret: config.npeClientSecret, PlatformEndpoint: config.platformEndpoint, TokenEndpoint: config.tokenEndpoint, KasUrl: config.kasEndpoint, - }, nil) + } + + got, err := gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt", ".csv"}, cfg, nil) if err != nil { - t.Fatal("Failed to EncryptFilesInDirNPE()!", err) + t.Fatal("Failed to EncryptFilesWithExtensionsNPE()!", err) } - if len(got) == 0 { - t.Fatal("EncryptFilesInDirNPE returned empty value, but didn't error!") + if len(got) != 3 { + t.Fatal("EncryptFilesWithExtensionsNPE returned incorrect got value, but didn't error!") } - fmt.Println("Successfully encrypted files in directory") + fmt.Println("Successfully encrypted files using file extensions") } -func Test_NPE_Encrypt_Files_Glob_Nil_Attributes(t *testing.T) { +// A new test of a new 'EncryptFilesWithExtensions' function +func Test_NPE_Encrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { // Create a temporary directory tmpDir, err := os.MkdirTemp("", "input-dir") if err != nil { @@ -429,8 +443,20 @@ func Test_NPE_Encrypt_Files_Glob_Nil_Attributes(t *testing.T) { t.Fatal("Unable to write to temporary file", err) } - // Call the EncryptFilesGlobNPE function - got, err := gotdf_python.EncryptFilesGlobNPE(tmpDir+"/*.txt", gotdf_python.OpentdfConfig{ + // Create a temporary file in the directory + tmpFile3, err := os.CreateTemp(tmpDir, "input-file3-*.csv") + if err != nil { + t.Fatal("Could not create input file", err) + } + defer tmpFile3.Close() + + // Write some data to the file + if _, err = tmpFile3.WriteString("test data"); err != nil { + t.Fatal("Unable to write to temporary file", err) + } + + // Call the EncryptFilesWithExtensionsNPE function + got, err := gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt", ".csv"}, gotdf_python.OpentdfConfig{ ClientId: config.npeClientId, ClientSecret: config.npeClientSecret, PlatformEndpoint: config.platformEndpoint, @@ -438,14 +464,14 @@ func Test_NPE_Encrypt_Files_Glob_Nil_Attributes(t *testing.T) { KasUrl: config.kasEndpoint, }, nil) if err != nil { - t.Fatal("Failed to EncryptFilesGlobNPE()!", err) + t.Fatal("Failed to EncryptFilesWithExtensionsNPE()!", err) } - if len(got) == 0 { - t.Fatal("EncryptFilesGlobNPE returned empty value, but didn't error!") + if len(got) != 3 { + t.Fatal("EncryptFilesWithExtensionsNPE returned incorrect got value, but didn't error!") } - fmt.Println("Successfully encrypted files using glob pattern") + fmt.Println("Successfully encrypted files with extensions") } // Call the DecryptFilesInDirNPE function @@ -512,7 +538,7 @@ func Test_NPE_Decrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { fmt.Println("Successfully decrypted files in directory") } -func Test_NPE_Decrypt_Files_Glob_Nil_Attributes(t *testing.T) { +func Test_NPE_Decrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { // Create a temporary directory tmpDir, err := os.MkdirTemp("", "input-dir") if err != nil { @@ -545,7 +571,7 @@ func Test_NPE_Decrypt_Files_Glob_Nil_Attributes(t *testing.T) { } // Encrypt the file - _, err = gotdf_python.EncryptFilesGlobNPE(tmpDir+"/*.txt", gotdf_python.OpentdfConfig{ + _, err = gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt"}, gotdf_python.OpentdfConfig{ ClientId: config.npeClientId, ClientSecret: config.npeClientSecret, PlatformEndpoint: config.platformEndpoint, @@ -553,11 +579,11 @@ func Test_NPE_Decrypt_Files_Glob_Nil_Attributes(t *testing.T) { KasUrl: config.kasEndpoint, }, nil) if err != nil { - t.Fatal("Failed to EncryptFilesGlobNPE()!", err) + t.Fatal("Failed to EncryptFilesWithExtensionsNPE()!", err) } - // Call the DecryptFilesGlobNPE function - got, err := gotdf_python.DecryptFilesGlobNPE(tmpDir+"/*.tdf", gotdf_python.OpentdfConfig{ + // Call the DecryptFilesWithExtensionsNPE function + got, err := gotdf_python.DecryptFilesWithExtensionsNPE(tmpDir, []string{".tdf"}, gotdf_python.OpentdfConfig{ ClientId: config.npeClientId, ClientSecret: config.npeClientSecret, PlatformEndpoint: config.platformEndpoint, @@ -565,12 +591,12 @@ func Test_NPE_Decrypt_Files_Glob_Nil_Attributes(t *testing.T) { KasUrl: config.kasEndpoint, }) if err != nil { - t.Fatal("Failed to DecryptFilesGlobNPE()!", err) + t.Fatal("Failed to DecryptFilesWithExtensionsNPE()!", err) } if len(got) == 0 { - t.Fatal("DecryptFilesGlobNPE returned empty value, but didn't error!") + t.Fatal("DecryptFilesWithExtensionsNPE returned empty value, but didn't error!") } - fmt.Println("Successfully decrypted files using glob pattern") + fmt.Println("Successfully decrypted files with extensions") } From b1d408b5806e7f968d39f961bcb549c0283fa4df Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 7 Feb 2025 16:55:38 -0500 Subject: [PATCH 06/19] Update tests --- otdf_python_test.go | 133 ++++++++++++++------------------------------ 1 file changed, 43 insertions(+), 90 deletions(-) diff --git a/otdf_python_test.go b/otdf_python_test.go index 95db6d0..3fcef8e 100644 --- a/otdf_python_test.go +++ b/otdf_python_test.go @@ -346,7 +346,7 @@ func Test_PE_E2E_File_Multi_Attributes(t *testing.T) { e2e_test_as_PE(t, attrValues) } -func Test_NPE_Encrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { +func Test_Multifile_NPE_Encrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { // Create a temporary directory tmpDir, err := os.MkdirTemp("", "input-dir") if err != nil { @@ -411,7 +411,7 @@ func Test_NPE_Encrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { } // A new test of a new 'EncryptFilesWithExtensions' function -func Test_NPE_Encrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { +func Test_Multifile_NPE_Encrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { // Create a temporary directory tmpDir, err := os.MkdirTemp("", "input-dir") if err != nil { @@ -419,41 +419,8 @@ func Test_NPE_Encrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { } defer os.RemoveAll(tmpDir) - // Create a temporary file in the directory - tmpFile1, err := os.CreateTemp(tmpDir, "input-file1-*.txt") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile1.Close() - - // Write some data to the file - if _, err = tmpFile1.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } - - // Create a temporary file in the directory - tmpFile2, err := os.CreateTemp(tmpDir, "input-file2-*.txt") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile2.Close() - - // Write some data to the file - if _, err = tmpFile2.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } - - // Create a temporary file in the directory - tmpFile3, err := os.CreateTemp(tmpDir, "input-file3-*.csv") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile3.Close() - - // Write some data to the file - if _, err = tmpFile3.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } + // Create test files + numFiles := createTestFiles(t, tmpDir) // Call the EncryptFilesWithExtensionsNPE function got, err := gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt", ".csv"}, gotdf_python.OpentdfConfig{ @@ -467,7 +434,7 @@ func Test_NPE_Encrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { t.Fatal("Failed to EncryptFilesWithExtensionsNPE()!", err) } - if len(got) != 3 { + if len(got) != numFiles { t.Fatal("EncryptFilesWithExtensionsNPE returned incorrect got value, but didn't error!") } @@ -475,7 +442,7 @@ func Test_NPE_Encrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { } // Call the DecryptFilesInDirNPE function -func Test_NPE_Decrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { +func Test_Multifile_NPE_Decrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { // Create a temporary directory tmpDir, err := os.MkdirTemp("", "input-dir") if err != nil { @@ -483,29 +450,8 @@ func Test_NPE_Decrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { } defer os.RemoveAll(tmpDir) - // Create a temporary file in the directory - tmpFile1, err := os.CreateTemp(tmpDir, "input-file1-*.txt") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile1.Close() - - // Write some data to the file - if _, err = tmpFile1.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } - - // Create a temporary file in the directory - tmpFile2, err := os.CreateTemp(tmpDir, "input-file2-*.txt") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile2.Close() - - // Write some data to the file - if _, err = tmpFile2.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } + // Create test files + numFiles := createTestFiles(t, tmpDir) // Encrypt the file _, err = gotdf_python.EncryptFilesInDirNPE(tmpDir, gotdf_python.OpentdfConfig{ @@ -531,14 +477,14 @@ func Test_NPE_Decrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { t.Fatal("Failed to DecryptFilesInDirNPE()!", err) } - if len(got) == 0 { + if len(got) != numFiles { t.Fatal("DecryptFilesInDirNPE returned empty value, but didn't error!") } fmt.Println("Successfully decrypted files in directory") } -func Test_NPE_Decrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { +func Test_Multifile_NPE_Decrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { // Create a temporary directory tmpDir, err := os.MkdirTemp("", "input-dir") if err != nil { @@ -546,32 +492,11 @@ func Test_NPE_Decrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { } defer os.RemoveAll(tmpDir) - // Create a temporary file in the directory - tmpFile1, err := os.CreateTemp(tmpDir, "input-file1-*.txt") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile1.Close() + // Create test files + numFiles := createTestFiles(t, tmpDir) - // Write some data to the file - if _, err = tmpFile1.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } - - // Create a temporary file in the directory - tmpFile2, err := os.CreateTemp(tmpDir, "input-file2-*.txt") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile2.Close() - - // Write some data to the file - if _, err = tmpFile2.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } - - // Encrypt the file - _, err = gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt"}, gotdf_python.OpentdfConfig{ + // Encrypt the files + _, err = gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt", ".csv"}, gotdf_python.OpentdfConfig{ ClientId: config.npeClientId, ClientSecret: config.npeClientSecret, PlatformEndpoint: config.platformEndpoint, @@ -594,9 +519,37 @@ func Test_NPE_Decrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { t.Fatal("Failed to DecryptFilesWithExtensionsNPE()!", err) } - if len(got) == 0 { + if len(got) != numFiles { t.Fatal("DecryptFilesWithExtensionsNPE returned empty value, but didn't error!") } fmt.Println("Successfully decrypted files with extensions") } + +func createTestFiles(t *testing.T, tmpDir string) int { + // A number that corresponds to the hour of the day (between 0 and 23) + numFiles := time.Now().Hour() + + if numFiles > 12 { + numFiles = numFiles - 12 // Limit the number of files to 12 + } + + for i := 0; i < numFiles; i++ { + ext := ".txt" + if i%2 == 0 { + ext = ".csv" + } + tmpFile, err := os.CreateTemp(tmpDir, fmt.Sprintf("input-file-%d-*%s", i, ext)) + if err != nil { + t.Fatal("Could not create input file", err) + } + defer tmpFile.Close() + + // Write some data to the file + if _, err = tmpFile.WriteString("test data"); err != nil { + t.Fatal("Unable to write to temporary file", err) + } + } + + return numFiles +} From 9cd3685d0a152ae011da371079770948cf3e4dca Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 7 Feb 2025 17:19:14 -0500 Subject: [PATCH 07/19] Cleanup & improve testing --- main.go | 58 +++++++++++++++++++++++++++++++-------------- otdf_python_test.go | 7 ++++-- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/main.go b/main.go index 5d435ab..3c89308 100644 --- a/main.go +++ b/main.go @@ -421,7 +421,7 @@ func EncryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O return nil, err } - files, err := os.ReadDir(dirPath) + files, err := findFiles(dirPath, extensions) if err != nil { return nil, err } @@ -429,20 +429,14 @@ func EncryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O var outputPaths []string var errors []error for _, file := range files { - if !file.IsDir() { - for _, ext := range extensions { - if strings.HasSuffix(file.Name(), ext) { - inputFilePath := filepath.Join(dirPath, file.Name()) - outputFilePath := inputFilePath + ".tdf" - got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) - if err != nil { - errors = append(errors, fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err)) - continue - } - outputPaths = append(outputPaths, got) - } - } + inputFilePath := file + outputFilePath := inputFilePath + ".tdf" + got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) + if err != nil { + errors = append(errors, fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err)) + continue } + outputPaths = append(outputPaths, got) } if len(errors) > 0 { @@ -575,10 +569,6 @@ in the same directory as the input files, with the .tdf extension removed from t */ func DecryptFilesInDirNPE(dirPath string, config OpentdfConfig) ([]string, error) { authScopes := []string{"email"} - sdkClient, err := newSdkClient(config, authScopes) - if err != nil { - return nil, err - } files, err := os.ReadDir(dirPath) if err != nil { @@ -594,6 +584,12 @@ func DecryptFilesInDirNPE(dirPath string, config OpentdfConfig) ([]string, error wg.Add(1) go func(file os.DirEntry) { defer wg.Done() + sdkClient, err := newSdkClient(config, authScopes) + if err != nil { + errChan <- fmt.Errorf("failed to create SDK client: %v", err) + return + } + fileInfo, err := file.Info() if err != nil { errChan <- fmt.Errorf("failed to get file info for %s: %v", file.Name(), err) @@ -796,3 +792,29 @@ func encryptBytesWithClient(b []byte, sdkClient *sdk.SDK, config OpentdfConfig, } return enc, nil } + +// Function to find all files recursively in a directory matching the given extensions +func findFiles(dir string, extensions []string) ([]string, error) { + var files []string + + // Use filepath.Walk to walk through the directory recursively + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + // If there's an error reading the file, skip it + return err + } + + // Check if the file extension matches 'extensions' parameter + if !info.IsDir() && strings.Contains(strings.Join(extensions, ","), filepath.Ext(path)) { + files = append(files, path) // Add the file to the list + } + + return nil + }) + + if err != nil { + return nil, err + } + + return files, nil +} diff --git a/otdf_python_test.go b/otdf_python_test.go index 3fcef8e..3fc835e 100644 --- a/otdf_python_test.go +++ b/otdf_python_test.go @@ -423,7 +423,7 @@ func Test_Multifile_NPE_Encrypt_Files_With_Extensions_Nil_Attributes(t *testing. numFiles := createTestFiles(t, tmpDir) // Call the EncryptFilesWithExtensionsNPE function - got, err := gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt", ".csv"}, gotdf_python.OpentdfConfig{ + got, err := gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt", ".csv", ".pdf"}, gotdf_python.OpentdfConfig{ ClientId: config.npeClientId, ClientSecret: config.npeClientSecret, PlatformEndpoint: config.platformEndpoint, @@ -496,7 +496,7 @@ func Test_Multifile_NPE_Decrypt_Files_With_Extensions_Nil_Attributes(t *testing. numFiles := createTestFiles(t, tmpDir) // Encrypt the files - _, err = gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt", ".csv"}, gotdf_python.OpentdfConfig{ + _, err = gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt", ".csv", ".pdf"}, gotdf_python.OpentdfConfig{ ClientId: config.npeClientId, ClientSecret: config.npeClientSecret, PlatformEndpoint: config.platformEndpoint, @@ -538,7 +538,10 @@ func createTestFiles(t *testing.T, tmpDir string) int { ext := ".txt" if i%2 == 0 { ext = ".csv" + } else if i%3 == 0 { + ext = ".pdf" } + tmpFile, err := os.CreateTemp(tmpDir, fmt.Sprintf("input-file-%d-*%s", i, ext)) if err != nil { t.Fatal("Could not create input file", err) From fe9f37098be4da19256311c4ab78c0dc3acc2f64 Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 7 Feb 2025 17:19:45 -0500 Subject: [PATCH 08/19] Update codespell --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37cf25c..7a278ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: # - id: validate-toml - id: no-go-testing - repo: https://github.com/codespell-project/codespell - rev: v2.3.0 + rev: v2.4.1 hooks: - id: codespell args: ["--ignore-words-list", "b-long, otdf_python", "--skip=go.sum,otdf_python/"] From 0de67b405350ea0fe73b838e0331d5e047c76869 Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 7 Feb 2025 17:20:59 -0500 Subject: [PATCH 09/19] Bump 0.2.6 -> 0.2.7 --- .github/workflows/build-golang-macos.yaml | 2 +- .github/workflows/build-golang-ubuntu.yaml | 6 +++--- .github/workflows/platform-integration-test.yaml | 2 +- README.md | 4 ++-- build-scripts/ci-build.sh | 2 +- build-scripts/make_and_validate_script.sh | 2 +- build-scripts/uv_make_and_validate_script.sh | 2 +- pyproject.toml | 4 ++-- setup.py | 2 +- setup_ci.py | 2 +- uv.lock | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index d1d165f..840e89d 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -47,4 +47,4 @@ jobs: # - uses: ./.github/workflows/platform-integration-test.yaml # with: - # wheel: dist/otdf_python-0.2.6-py3-none-any.whl + # wheel: dist/otdf_python-0.2.7-py3-none-any.whl diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index a98b148..e1d844c 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -43,12 +43,12 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.6-py3-none-any.whl + path: dist/otdf_python-0.2.7-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} - uses: actions/cache/save@v4 with: - path: dist/otdf_python-0.2.6-py3-none-any.whl + path: dist/otdf_python-0.2.7-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} restore-keys: | ${{ runner.os }}${{ matrix.python3_version }}-data- @@ -61,5 +61,5 @@ jobs: needs: build uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.2.6-py3-none-any.whl + wheel: dist/otdf_python-0.2.7-py3-none-any.whl python_version: ${{ matrix.python3_version }} diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index e80a861..ec58f2f 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.6-py3-none-any.whl + path: dist/otdf_python-0.2.7-py3-none-any.whl key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }} - name: Prove that the input file is available diff --git a/README.md b/README.md index ae9752b..7974091 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,10 @@ Install from the [Python Package Index (PyPI)](https://pypi.org): pip install otdf_python # Install a pinned version -pip install otdf-python==0.2.6 +pip install otdf-python==0.2.7 # Install a pinned version, from test.pypi.org -pip install -i https://test.pypi.org/simple/ otdf-python==0.2.6 +pip install -i https://test.pypi.org/simple/ otdf-python==0.2.7 ``` ## Usage diff --git a/build-scripts/ci-build.sh b/build-scripts/ci-build.sh index 394e377..cc54576 100755 --- a/build-scripts/ci-build.sh +++ b/build-scripts/ci-build.sh @@ -72,4 +72,4 @@ echo "✨✨✨ Build wheel" poetry run python3 setup.py bdist_wheel echo "✨✨✨ Install wheel" -pip install dist/otdf_python-0.2.6-py3-none-any.whl +pip install dist/otdf_python-0.2.7-py3-none-any.whl diff --git a/build-scripts/make_and_validate_script.sh b/build-scripts/make_and_validate_script.sh index b151ccd..06369d8 100755 --- a/build-scripts/make_and_validate_script.sh +++ b/build-scripts/make_and_validate_script.sh @@ -47,7 +47,7 @@ python3 -m pip install --upgrade setuptools wheel python3 setup.py bdist_wheel # Prove that the wheel can be installed -pip install dist/otdf_python-0.2.6-py3-none-any.whl +pip install dist/otdf_python-0.2.7-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/build-scripts/uv_make_and_validate_script.sh b/build-scripts/uv_make_and_validate_script.sh index d9bef3b..52f48c3 100755 --- a/build-scripts/uv_make_and_validate_script.sh +++ b/build-scripts/uv_make_and_validate_script.sh @@ -70,7 +70,7 @@ loud_print "Installing wheel" uv venv .venv-wheel --python 3.12 "$PY_TYPE" source "${BUILD_ROOT}/.venv-wheel/bin/activate" pip install pybindgen -pip install dist/otdf_python-0.2.6-py3-none-any.whl +pip install dist/otdf_python-0.2.7-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/pyproject.toml b/pyproject.toml index 7ab6610..9f5b784 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "otdf-python" # Should match 'setup.py' version number (used for gopy/pybindgen) -version = "0.2.6" +version = "0.2.7" description = "Unofficial OpenTDF SDK for Python." authors = [ {name="b-long", email="b-long@users.noreply.github.com"} @@ -19,7 +19,7 @@ pybindgen = "^0.22.1" [tool.poetry] package-mode = false -version = "0.2.6" +version = "0.2.7" [tool.poetry.dependencies] python = ">=3.11,<3.14" diff --git a/setup.py b/setup.py index c270834..334e729 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url="https://github.com/b-long/opentdf-python-sdk", package_data={"otdf_python": ["*.so"]}, # Should match 'pyproject.toml' version number - version="0.2.6", + version="0.2.7", author_email="b-long@users.noreply.github.com", include_package_data=True, ) diff --git a/setup_ci.py b/setup_ci.py index a602293..7171999 100644 --- a/setup_ci.py +++ b/setup_ci.py @@ -81,7 +81,7 @@ def build_extension(self, ext: Extension): setuptools.setup( name="otdf_python", - version="0.2.6", + version="0.2.7", author="b-long", description="Unofficial OpenTDF SDK for Python.", long_description_content_type="text/markdown", diff --git a/uv.lock b/uv.lock index 72e576a..8b0cbf0 100644 --- a/uv.lock +++ b/uv.lock @@ -3,5 +3,5 @@ requires-python = ">=3.11" [[package]] name = "otdf-python" -version = "0.2.6" +version = "0.2.7" source = { editable = "." } From 7c860e442a4691bc54ed9d87341a1dc2bd2bcf09 Mon Sep 17 00:00:00 2001 From: b-long Date: Mon, 10 Feb 2025 19:49:47 -0500 Subject: [PATCH 10/19] More logging --- main.go | 45 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 3c89308..c847dea 100644 --- a/main.go +++ b/main.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "log" "os" "path" "path/filepath" @@ -374,6 +375,7 @@ func EncryptFilesInDirNPE(dirPath string, config OpentdfConfig, dataAttributes [ if err != nil { return nil, err } + errChan := make(chan error, len(files)) var outputPaths []string var mu sync.Mutex @@ -386,14 +388,14 @@ func EncryptFilesInDirNPE(dirPath string, config OpentdfConfig, dataAttributes [ defer wg.Done() sdkClient, err := newSdkClient(config, authScopes) if err != nil { - fmt.Printf("failed to create SDK client: %v\n", err) + errChan <- fmt.Errorf("failed to create SDK client: %v", err) return } inputFilePath := path.Join(dirPath, file.Name()) outputFilePath := inputFilePath + ".tdf" got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) if err != nil { - fmt.Printf("failed to encrypt file %s: %v\n", inputFilePath, err) + errChan <- fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err) return } mu.Lock() @@ -404,7 +406,18 @@ func EncryptFilesInDirNPE(dirPath string, config OpentdfConfig, dataAttributes [ } wg.Wait() + close(errChan) + + var errors []error + for err := range errChan { + errors = append(errors, err) + } + logOutputPaths(outputPaths, errors) + + if len(errors) > 0 { + return outputPaths, fmt.Errorf("encountered errors during encryption: %v", errors) + } return outputPaths, nil } @@ -439,6 +452,8 @@ func EncryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O outputPaths = append(outputPaths, got) } + logOutputPaths(outputPaths, errors) + if len(errors) > 0 { return outputPaths, fmt.Errorf("encountered errors during encryption: %v", errors) } @@ -637,8 +652,15 @@ func DecryptFilesInDirNPE(dirPath string, config OpentdfConfig) ([]string, error outputPaths = append(outputPaths, path) } - if len(errChan) > 0 { - return nil, <-errChan + var errors []error + for err := range errChan { + errors = append(errors, err) + } + + logOutputPaths(outputPaths, errors) + + if len(errors) > 0 { + return nil, fmt.Errorf("encountered errors during decryption: %v", errors) } return outputPaths, nil @@ -701,6 +723,8 @@ func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O } } + logOutputPaths(outputPaths, errors) + if len(errors) > 0 { return outputPaths, fmt.Errorf("encountered errors during decryption: %v", errors) } @@ -818,3 +842,16 @@ func findFiles(dir string, extensions []string) ([]string, error) { return files, nil } + +func logOutputPaths(outputPaths []string, errors []error) { + if len(errors) > 0 { + log.Println("Errors occurred during processing:") + for _, err := range errors { + log.Printf("\t%s\n", err) + } + } + log.Println("Output Paths:") + for _, path := range outputPaths { + log.Printf("\t%s\n", path) + } +} From cd9e41bef77e7264f129e583a5719e603fcd4dad Mon Sep 17 00:00:00 2001 From: b-long Date: Mon, 10 Feb 2025 19:56:47 -0500 Subject: [PATCH 11/19] Bump 0.2.7 -> 0.2.8 --- .github/workflows/build-golang-macos.yaml | 2 +- .github/workflows/build-golang-ubuntu.yaml | 6 +++--- .github/workflows/platform-integration-test.yaml | 2 +- README.md | 4 ++-- build-scripts/ci-build.sh | 2 +- build-scripts/make_and_validate_script.sh | 2 +- build-scripts/uv_make_and_validate_script.sh | 2 +- pyproject.toml | 4 ++-- setup.py | 2 +- setup_ci.py | 2 +- uv.lock | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index 840e89d..86ad73b 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -47,4 +47,4 @@ jobs: # - uses: ./.github/workflows/platform-integration-test.yaml # with: - # wheel: dist/otdf_python-0.2.7-py3-none-any.whl + # wheel: dist/otdf_python-0.2.8-py3-none-any.whl diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index e1d844c..2a004cd 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -43,12 +43,12 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.7-py3-none-any.whl + path: dist/otdf_python-0.2.8-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} - uses: actions/cache/save@v4 with: - path: dist/otdf_python-0.2.7-py3-none-any.whl + path: dist/otdf_python-0.2.8-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} restore-keys: | ${{ runner.os }}${{ matrix.python3_version }}-data- @@ -61,5 +61,5 @@ jobs: needs: build uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.2.7-py3-none-any.whl + wheel: dist/otdf_python-0.2.8-py3-none-any.whl python_version: ${{ matrix.python3_version }} diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index ec58f2f..a042053 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.7-py3-none-any.whl + path: dist/otdf_python-0.2.8-py3-none-any.whl key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }} - name: Prove that the input file is available diff --git a/README.md b/README.md index 7974091..a4eb971 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,10 @@ Install from the [Python Package Index (PyPI)](https://pypi.org): pip install otdf_python # Install a pinned version -pip install otdf-python==0.2.7 +pip install otdf-python==0.2.8 # Install a pinned version, from test.pypi.org -pip install -i https://test.pypi.org/simple/ otdf-python==0.2.7 +pip install -i https://test.pypi.org/simple/ otdf-python==0.2.8 ``` ## Usage diff --git a/build-scripts/ci-build.sh b/build-scripts/ci-build.sh index cc54576..8158e11 100755 --- a/build-scripts/ci-build.sh +++ b/build-scripts/ci-build.sh @@ -72,4 +72,4 @@ echo "✨✨✨ Build wheel" poetry run python3 setup.py bdist_wheel echo "✨✨✨ Install wheel" -pip install dist/otdf_python-0.2.7-py3-none-any.whl +pip install dist/otdf_python-0.2.8-py3-none-any.whl diff --git a/build-scripts/make_and_validate_script.sh b/build-scripts/make_and_validate_script.sh index 06369d8..b49c8f1 100755 --- a/build-scripts/make_and_validate_script.sh +++ b/build-scripts/make_and_validate_script.sh @@ -47,7 +47,7 @@ python3 -m pip install --upgrade setuptools wheel python3 setup.py bdist_wheel # Prove that the wheel can be installed -pip install dist/otdf_python-0.2.7-py3-none-any.whl +pip install dist/otdf_python-0.2.8-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/build-scripts/uv_make_and_validate_script.sh b/build-scripts/uv_make_and_validate_script.sh index 52f48c3..2d2a374 100755 --- a/build-scripts/uv_make_and_validate_script.sh +++ b/build-scripts/uv_make_and_validate_script.sh @@ -70,7 +70,7 @@ loud_print "Installing wheel" uv venv .venv-wheel --python 3.12 "$PY_TYPE" source "${BUILD_ROOT}/.venv-wheel/bin/activate" pip install pybindgen -pip install dist/otdf_python-0.2.7-py3-none-any.whl +pip install dist/otdf_python-0.2.8-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/pyproject.toml b/pyproject.toml index 9f5b784..676d80f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "otdf-python" # Should match 'setup.py' version number (used for gopy/pybindgen) -version = "0.2.7" +version = "0.2.8" description = "Unofficial OpenTDF SDK for Python." authors = [ {name="b-long", email="b-long@users.noreply.github.com"} @@ -19,7 +19,7 @@ pybindgen = "^0.22.1" [tool.poetry] package-mode = false -version = "0.2.7" +version = "0.2.8" [tool.poetry.dependencies] python = ">=3.11,<3.14" diff --git a/setup.py b/setup.py index 334e729..95dc039 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url="https://github.com/b-long/opentdf-python-sdk", package_data={"otdf_python": ["*.so"]}, # Should match 'pyproject.toml' version number - version="0.2.7", + version="0.2.8", author_email="b-long@users.noreply.github.com", include_package_data=True, ) diff --git a/setup_ci.py b/setup_ci.py index 7171999..5bb75af 100644 --- a/setup_ci.py +++ b/setup_ci.py @@ -81,7 +81,7 @@ def build_extension(self, ext: Extension): setuptools.setup( name="otdf_python", - version="0.2.7", + version="0.2.8", author="b-long", description="Unofficial OpenTDF SDK for Python.", long_description_content_type="text/markdown", diff --git a/uv.lock b/uv.lock index 8b0cbf0..138c19b 100644 --- a/uv.lock +++ b/uv.lock @@ -3,5 +3,5 @@ requires-python = ">=3.11" [[package]] name = "otdf-python" -version = "0.2.7" +version = "0.2.8" source = { editable = "." } From 1742f85123a660a47b9051dfafd02326451aebd6 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 11 Feb 2025 17:15:34 -0500 Subject: [PATCH 12/19] Initialize with a size for speed --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index c847dea..542814d 100644 --- a/main.go +++ b/main.go @@ -439,8 +439,8 @@ func EncryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O return nil, err } - var outputPaths []string - var errors []error + var outputPaths = make([]string, 0, len(files)) + var errors = make([]error, 0, len(files)) for _, file := range files { inputFilePath := file outputFilePath := inputFilePath + ".tdf" From 4bff87ec03d06c88a3047eb4aef5e689051bf66d Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 11 Feb 2025 17:16:25 -0500 Subject: [PATCH 13/19] Provide more error detail --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 542814d..8de7d89 100644 --- a/main.go +++ b/main.go @@ -726,7 +726,7 @@ func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O logOutputPaths(outputPaths, errors) if len(errors) > 0 { - return outputPaths, fmt.Errorf("encountered errors during decryption: %v", errors) + return outputPaths, fmt.Errorf("encountered errors during decryption of files in directory %s: %v", dirPath, errors) } return outputPaths, nil } From 2faddcbece080e76d8e870519d74aa649a8ff2e4 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 11 Feb 2025 17:18:00 -0500 Subject: [PATCH 14/19] Provide comment for 'logOutputPaths' --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 8de7d89..95639f7 100644 --- a/main.go +++ b/main.go @@ -843,6 +843,7 @@ func findFiles(dir string, extensions []string) ([]string, error) { return files, nil } +// logOutputPaths logs the output paths and any errors that occurred during processing func logOutputPaths(outputPaths []string, errors []error) { if len(errors) > 0 { log.Println("Errors occurred during processing:") From 4eab9cfc4110a6159e5b9e181d86421509dc8b4d Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 11 Feb 2025 17:30:45 -0500 Subject: [PATCH 15/19] Improve return values/logic --- main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 95639f7..748a259 100644 --- a/main.go +++ b/main.go @@ -725,8 +725,11 @@ func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O logOutputPaths(outputPaths, errors) - if len(errors) > 0 { - return outputPaths, fmt.Errorf("encountered errors during decryption of files in directory %s: %v", dirPath, errors) + if len(outputPaths) == 0 { + if len(errors) == 0 { + return nil, fmt.Errorf("no files with extensions %v found in directory %s", extensions, dirPath) + } + return nil, fmt.Errorf("encountered errors during decryption of files in directory %s: %v", dirPath, errors) } return outputPaths, nil } From 44ff880969442807861b9fcc807f2c6b2178f60c Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 11 Feb 2025 17:32:01 -0500 Subject: [PATCH 16/19] Initialize with a size for speed --- main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 748a259..446f145 100644 --- a/main.go +++ b/main.go @@ -683,8 +683,9 @@ func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config O return nil, err } - var outputPaths []string - var errors []error + var outputPaths = make([]string, 0, len(files)) + var errors = make([]error, 0, len(files)) + for _, file := range files { if !file.IsDir() { for _, ext := range extensions { From 2ba4f60c80f4fdc7d68bcf4585d4027612cedfe7 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 11 Feb 2025 17:39:11 -0500 Subject: [PATCH 17/19] Use newSdkClient in loop & use channels for data --- main.go | 92 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/main.go b/main.go index 446f145..5010cc3 100644 --- a/main.go +++ b/main.go @@ -673,57 +673,79 @@ in the same directory as the input files, with the .tdf extension removed from t */ func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config OpentdfConfig) ([]string, error) { authScopes := []string{"email"} - sdkClient, err := newSdkClient(config, authScopes) - if err != nil { - return nil, err - } files, err := os.ReadDir(dirPath) if err != nil { return nil, err } - var outputPaths = make([]string, 0, len(files)) - var errors = make([]error, 0, len(files)) + outputPathsChan := make(chan string, len(files)) + errChan := make(chan error, len(files)) + + var wg sync.WaitGroup for _, file := range files { if !file.IsDir() { for _, ext := range extensions { if strings.HasSuffix(file.Name(), ext) { - inputFilePath := filepath.Join(dirPath, file.Name()) - outputFilePath := strings.TrimSuffix(inputFilePath, ext) - - bytes, err := readBytesFromFile(inputFilePath) - if err != nil { - errors = append(errors, fmt.Errorf("failed to read file %s: %v", inputFilePath, err)) - continue - } - - decrypted, err := decryptBytesWithClient(bytes, sdkClient) - if err != nil { - errors = append(errors, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err)) - continue - } - - tdfFile, err := os.Create(outputFilePath) - if err != nil { - errors = append(errors, fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err)) - continue - } - defer tdfFile.Close() - - _, e := io.Copy(tdfFile, decrypted) - if e != nil { - errors = append(errors, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err)) - continue - } - - outputPaths = append(outputPaths, outputFilePath) + wg.Add(1) + go func(file os.DirEntry, ext string) { + defer wg.Done() + sdkClient, err := newSdkClient(config, authScopes) + if err != nil { + errChan <- fmt.Errorf("failed to create SDK client: %v", err) + return + } + + inputFilePath := filepath.Join(dirPath, file.Name()) + outputFilePath := strings.TrimSuffix(inputFilePath, ext) + + bytes, err := readBytesFromFile(inputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to read file %s: %v", inputFilePath, err) + return + } + + decrypted, err := decryptBytesWithClient(bytes, sdkClient) + if err != nil { + errChan <- fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err) + return + } + + tdfFile, err := os.Create(outputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err) + return + } + defer tdfFile.Close() + + _, e := io.Copy(tdfFile, decrypted) + if e != nil { + errChan <- fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err) + return + } + + outputPathsChan <- outputFilePath + }(file, ext) } } } } + wg.Wait() + close(outputPathsChan) + close(errChan) + + var outputPaths []string + for path := range outputPathsChan { + outputPaths = append(outputPaths, path) + } + + var errors []error + for err := range errChan { + errors = append(errors, err) + } + logOutputPaths(outputPaths, errors) if len(outputPaths) == 0 { From c95e1dcce65b2315a8060ea1700f0683da62c84f Mon Sep 17 00:00:00 2001 From: b-long Date: Wed, 12 Feb 2025 09:25:34 -0500 Subject: [PATCH 18/19] Add timeout to ARM builds --- .github/workflows/publish-test.yaml | 1 + .github/workflows/publish.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/publish-test.yaml b/.github/workflows/publish-test.yaml index 7cb73c3..af9bebb 100644 --- a/.github/workflows/publish-test.yaml +++ b/.github/workflows/publish-test.yaml @@ -127,6 +127,7 @@ jobs: build_linux_arm: name: Linux Python ARM runs-on: ubuntu-22.04 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 457183a..ca73a26 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -127,6 +127,7 @@ jobs: build_linux_arm: name: Linux Python ARM runs-on: ubuntu-22.04 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 From 18525f65a99d4c9bd7da6d516837fe7e652114e4 Mon Sep 17 00:00:00 2001 From: b-long Date: Wed, 12 Feb 2025 09:26:26 -0500 Subject: [PATCH 19/19] Bump 0.2.8 -> 0.2.9 --- .github/workflows/build-golang-macos.yaml | 2 +- .github/workflows/build-golang-ubuntu.yaml | 6 +++--- .github/workflows/platform-integration-test.yaml | 2 +- README.md | 4 ++-- build-scripts/ci-build.sh | 2 +- build-scripts/make_and_validate_script.sh | 2 +- build-scripts/uv_make_and_validate_script.sh | 2 +- go.sum | 4 ++-- pyproject.toml | 4 ++-- setup.py | 2 +- setup_ci.py | 2 +- uv.lock | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index 86ad73b..31a2e25 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -47,4 +47,4 @@ jobs: # - uses: ./.github/workflows/platform-integration-test.yaml # with: - # wheel: dist/otdf_python-0.2.8-py3-none-any.whl + # wheel: dist/otdf_python-0.2.9-py3-none-any.whl diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index 2a004cd..5e1c0db 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -43,12 +43,12 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.8-py3-none-any.whl + path: dist/otdf_python-0.2.9-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} - uses: actions/cache/save@v4 with: - path: dist/otdf_python-0.2.8-py3-none-any.whl + path: dist/otdf_python-0.2.9-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} restore-keys: | ${{ runner.os }}${{ matrix.python3_version }}-data- @@ -61,5 +61,5 @@ jobs: needs: build uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.2.8-py3-none-any.whl + wheel: dist/otdf_python-0.2.9-py3-none-any.whl python_version: ${{ matrix.python3_version }} diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index a042053..1d7b5d0 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.8-py3-none-any.whl + path: dist/otdf_python-0.2.9-py3-none-any.whl key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }} - name: Prove that the input file is available diff --git a/README.md b/README.md index a4eb971..42c96eb 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,10 @@ Install from the [Python Package Index (PyPI)](https://pypi.org): pip install otdf_python # Install a pinned version -pip install otdf-python==0.2.8 +pip install otdf-python==0.2.9 # Install a pinned version, from test.pypi.org -pip install -i https://test.pypi.org/simple/ otdf-python==0.2.8 +pip install -i https://test.pypi.org/simple/ otdf-python==0.2.9 ``` ## Usage diff --git a/build-scripts/ci-build.sh b/build-scripts/ci-build.sh index 8158e11..4584290 100755 --- a/build-scripts/ci-build.sh +++ b/build-scripts/ci-build.sh @@ -72,4 +72,4 @@ echo "✨✨✨ Build wheel" poetry run python3 setup.py bdist_wheel echo "✨✨✨ Install wheel" -pip install dist/otdf_python-0.2.8-py3-none-any.whl +pip install dist/otdf_python-0.2.9-py3-none-any.whl diff --git a/build-scripts/make_and_validate_script.sh b/build-scripts/make_and_validate_script.sh index b49c8f1..62f6b83 100755 --- a/build-scripts/make_and_validate_script.sh +++ b/build-scripts/make_and_validate_script.sh @@ -47,7 +47,7 @@ python3 -m pip install --upgrade setuptools wheel python3 setup.py bdist_wheel # Prove that the wheel can be installed -pip install dist/otdf_python-0.2.8-py3-none-any.whl +pip install dist/otdf_python-0.2.9-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/build-scripts/uv_make_and_validate_script.sh b/build-scripts/uv_make_and_validate_script.sh index 2d2a374..e9f6ab6 100755 --- a/build-scripts/uv_make_and_validate_script.sh +++ b/build-scripts/uv_make_and_validate_script.sh @@ -70,7 +70,7 @@ loud_print "Installing wheel" uv venv .venv-wheel --python 3.12 "$PY_TYPE" source "${BUILD_ROOT}/.venv-wheel/bin/activate" pip install pybindgen -pip install dist/otdf_python-0.2.8-py3-none-any.whl +pip install dist/otdf_python-0.2.9-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/go.sum b/go.sum index 33b6f01..4d08aea 100644 --- a/go.sum +++ b/go.sum @@ -99,8 +99,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opentdf/platform/lib/fixtures v0.2.8 h1:lGYrMnbORtU62lxsJi8qPsxjFuNIkc4Dop8rVkH6pD0= -github.com/opentdf/platform/lib/fixtures v0.2.8/go.mod h1:8yCSe+oUzW9jbM573r9qgE68rjwDMNzktObiGVsO/W8= +github.com/opentdf/platform/lib/fixtures v0.2.9 h1:lGYrMnbORtU62lxsJi8qPsxjFuNIkc4Dop8rVkH6pD0= +github.com/opentdf/platform/lib/fixtures v0.2.9/go.mod h1:8yCSe+oUzW9jbM573r9qgE68rjwDMNzktObiGVsO/W8= github.com/opentdf/platform/lib/ocrypto v0.1.7 h1:IcCYRrwmMqntqUE8frmUDg5EZ0WMdldpGeGhbv9+/A8= github.com/opentdf/platform/lib/ocrypto v0.1.7/go.mod h1:4bhKPbRFzURMerH5Vr/LlszHvcoXQbfJXa0bpY7/7yg= github.com/opentdf/platform/protocol/go v0.2.22 h1:C/jjtwu5yTon8g0ewuN29QE7VXSQHyb2dx9W0U6Oqok= diff --git a/pyproject.toml b/pyproject.toml index 676d80f..3081133 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "otdf-python" # Should match 'setup.py' version number (used for gopy/pybindgen) -version = "0.2.8" +version = "0.2.9" description = "Unofficial OpenTDF SDK for Python." authors = [ {name="b-long", email="b-long@users.noreply.github.com"} @@ -19,7 +19,7 @@ pybindgen = "^0.22.1" [tool.poetry] package-mode = false -version = "0.2.8" +version = "0.2.9" [tool.poetry.dependencies] python = ">=3.11,<3.14" diff --git a/setup.py b/setup.py index 95dc039..a140562 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url="https://github.com/b-long/opentdf-python-sdk", package_data={"otdf_python": ["*.so"]}, # Should match 'pyproject.toml' version number - version="0.2.8", + version="0.2.9", author_email="b-long@users.noreply.github.com", include_package_data=True, ) diff --git a/setup_ci.py b/setup_ci.py index 5bb75af..bfac080 100644 --- a/setup_ci.py +++ b/setup_ci.py @@ -81,7 +81,7 @@ def build_extension(self, ext: Extension): setuptools.setup( name="otdf_python", - version="0.2.8", + version="0.2.9", author="b-long", description="Unofficial OpenTDF SDK for Python.", long_description_content_type="text/markdown", diff --git a/uv.lock b/uv.lock index 138c19b..59938cf 100644 --- a/uv.lock +++ b/uv.lock @@ -3,5 +3,5 @@ requires-python = ">=3.11" [[package]] name = "otdf-python" -version = "0.2.8" +version = "0.2.9" source = { editable = "." }