diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml index 86910bc..31a2e25 100644 --- a/.github/workflows/build-golang-macos.yaml +++ b/.github/workflows/build-golang-macos.yaml @@ -47,4 +47,4 @@ jobs: # - uses: ./.github/workflows/platform-integration-test.yaml # with: - # wheel: dist/otdf_python-0.2.4-py3-none-any.whl + # wheel: dist/otdf_python-0.2.9-py3-none-any.whl diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml index b7b3122..5e1c0db 100644 --- a/.github/workflows/build-golang-ubuntu.yaml +++ b/.github/workflows/build-golang-ubuntu.yaml @@ -43,12 +43,12 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.4-py3-none-any.whl + path: dist/otdf_python-0.2.9-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} - uses: actions/cache/save@v4 with: - path: dist/otdf_python-0.2.4-py3-none-any.whl + path: dist/otdf_python-0.2.9-py3-none-any.whl key: ${{ runner.os }}${{ matrix.python3_version }}-data-${{ github.sha }} restore-keys: | ${{ runner.os }}${{ matrix.python3_version }}-data- @@ -61,5 +61,5 @@ jobs: needs: build uses: ./.github/workflows/platform-integration-test.yaml with: - wheel: dist/otdf_python-0.2.4-py3-none-any.whl + wheel: dist/otdf_python-0.2.9-py3-none-any.whl python_version: ${{ matrix.python3_version }} diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index b8619fb..1d7b5d0 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/cache/restore@v4 with: - path: dist/otdf_python-0.2.4-py3-none-any.whl + path: dist/otdf_python-0.2.9-py3-none-any.whl key: ${{ runner.os }}${{ inputs.python_version }}-data-${{ github.sha }} - name: Prove that the input file is available diff --git a/.github/workflows/publish-test.yaml b/.github/workflows/publish-test.yaml index 7cb73c3..af9bebb 100644 --- a/.github/workflows/publish-test.yaml +++ b/.github/workflows/publish-test.yaml @@ -127,6 +127,7 @@ jobs: build_linux_arm: name: Linux Python ARM runs-on: ubuntu-22.04 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 457183a..ca73a26 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -127,6 +127,7 @@ jobs: build_linux_arm: name: Linux Python ARM runs-on: ubuntu-22.04 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37cf25c..7a278ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: # - id: validate-toml - id: no-go-testing - repo: https://github.com/codespell-project/codespell - rev: v2.3.0 + rev: v2.4.1 hooks: - id: codespell args: ["--ignore-words-list", "b-long, otdf_python", "--skip=go.sum,otdf_python/"] diff --git a/README.md b/README.md index 74c5bfd..42c96eb 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,10 @@ Install from the [Python Package Index (PyPI)](https://pypi.org): pip install otdf_python # Install a pinned version -pip install otdf-python==0.2.4 +pip install otdf-python==0.2.9 # Install a pinned version, from test.pypi.org -pip install -i https://test.pypi.org/simple/ otdf-python==0.2.4 +pip install -i https://test.pypi.org/simple/ otdf-python==0.2.9 ``` ## Usage diff --git a/build-scripts/ci-build.sh b/build-scripts/ci-build.sh index 6fc7090..4584290 100755 --- a/build-scripts/ci-build.sh +++ b/build-scripts/ci-build.sh @@ -72,4 +72,4 @@ echo "✨✨✨ Build wheel" poetry run python3 setup.py bdist_wheel echo "✨✨✨ Install wheel" -pip install dist/otdf_python-0.2.4-py3-none-any.whl +pip install dist/otdf_python-0.2.9-py3-none-any.whl diff --git a/build-scripts/make_and_validate_script.sh b/build-scripts/make_and_validate_script.sh index e6791cb..62f6b83 100755 --- a/build-scripts/make_and_validate_script.sh +++ b/build-scripts/make_and_validate_script.sh @@ -47,7 +47,7 @@ python3 -m pip install --upgrade setuptools wheel python3 setup.py bdist_wheel # Prove that the wheel can be installed -pip install dist/otdf_python-0.2.4-py3-none-any.whl +pip install dist/otdf_python-0.2.9-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/build-scripts/uv_make_and_validate_script.sh b/build-scripts/uv_make_and_validate_script.sh index 34f11b4..e9f6ab6 100755 --- a/build-scripts/uv_make_and_validate_script.sh +++ b/build-scripts/uv_make_and_validate_script.sh @@ -70,7 +70,7 @@ loud_print "Installing wheel" uv venv .venv-wheel --python 3.12 "$PY_TYPE" source "${BUILD_ROOT}/.venv-wheel/bin/activate" pip install pybindgen -pip install dist/otdf_python-0.2.4-py3-none-any.whl +pip install dist/otdf_python-0.2.9-py3-none-any.whl if [[ "$SKIP_TESTS" == "-s" || "$SKIP_TESTS" == "--skip-tests" ]]; then echo "Build is complete, skipping tests." diff --git a/go.sum b/go.sum index 33b6f01..4d08aea 100644 --- a/go.sum +++ b/go.sum @@ -99,8 +99,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opentdf/platform/lib/fixtures v0.2.8 h1:lGYrMnbORtU62lxsJi8qPsxjFuNIkc4Dop8rVkH6pD0= -github.com/opentdf/platform/lib/fixtures v0.2.8/go.mod h1:8yCSe+oUzW9jbM573r9qgE68rjwDMNzktObiGVsO/W8= +github.com/opentdf/platform/lib/fixtures v0.2.9 h1:lGYrMnbORtU62lxsJi8qPsxjFuNIkc4Dop8rVkH6pD0= +github.com/opentdf/platform/lib/fixtures v0.2.9/go.mod h1:8yCSe+oUzW9jbM573r9qgE68rjwDMNzktObiGVsO/W8= github.com/opentdf/platform/lib/ocrypto v0.1.7 h1:IcCYRrwmMqntqUE8frmUDg5EZ0WMdldpGeGhbv9+/A8= github.com/opentdf/platform/lib/ocrypto v0.1.7/go.mod h1:4bhKPbRFzURMerH5Vr/LlszHvcoXQbfJXa0bpY7/7yg= github.com/opentdf/platform/protocol/go v0.2.22 h1:C/jjtwu5yTon8g0ewuN29QE7VXSQHyb2dx9W0U6Oqok= diff --git a/main.go b/main.go index d8050ec..5010cc3 100644 --- a/main.go +++ b/main.go @@ -2,12 +2,17 @@ package gotdf_python /* All public (upper-case) functions here should be available to Python. +* E.g. imported & tested via 'validate_otdf_python.py' -As a result, all public functions should be imported & tested. +TODO: Consider testing against attributes that are returned by some listing. +* See: https://github.com/orgs/opentdf/discussions/947 -Currently, testing is performed via 'validate_otdf_python.py' +TODO: Consider exposing an sdkClient that can be returned to the caller +* Note, previously this failed in a 'gopy' compiled context -FIXME: Expand test coverage, with known good attributes. See: https://github.com/orgs/opentdf/discussions/947 +TODO: Platform knows about the IdP, perhaps we don't need to specify the TOKEN_ENDPOINT ? + +TODO: Expose authScopes []string (such as []string{"email"}) parameter to various functions */ import ( @@ -16,10 +21,12 @@ import ( "errors" "fmt" "io" + "log" "os" "path" "path/filepath" "strings" + "sync" "github.com/opentdf/platform/sdk" ) @@ -363,60 +370,92 @@ in the same directory as the input files, with a .tdf extension added to the fil */ func EncryptFilesInDirNPE(dirPath string, config OpentdfConfig, dataAttributes []string) ([]string, error) { authScopes := []string{"email"} - sdkClient, err := newSdkClient(config, authScopes) - if err != nil { - return nil, err - } files, err := os.ReadDir(dirPath) if err != nil { return nil, err } + errChan := make(chan error, len(files)) var outputPaths []string + var mu sync.Mutex + var wg sync.WaitGroup + for _, file := range files { if !file.IsDir() { - inputFilePath := path.Join(dirPath, file.Name()) - outputFilePath := inputFilePath + ".tdf" - got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) - if err != nil { - return nil, fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err) - } else { + wg.Add(1) + go func(file os.DirEntry) { + defer wg.Done() + sdkClient, err := newSdkClient(config, authScopes) + if err != nil { + errChan <- fmt.Errorf("failed to create SDK client: %v", err) + return + } + inputFilePath := path.Join(dirPath, file.Name()) + outputFilePath := inputFilePath + ".tdf" + got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) + if err != nil { + errChan <- fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err) + return + } + mu.Lock() outputPaths = append(outputPaths, got) - } + mu.Unlock() + }(file) } } + + wg.Wait() + close(errChan) + + var errors []error + for err := range errChan { + errors = append(errors, err) + } + + logOutputPaths(outputPaths, errors) + + if len(errors) > 0 { + return outputPaths, fmt.Errorf("encountered errors during encryption: %v", errors) + } return outputPaths, nil } /* - EncryptFilesGlobNPE encrypts all files matching the specified glob pattern. + EncryptFilesWithExtensionsNPE encrypts all files in 'dirPath' with given file 'extensions'. Work is performed as an NPE (Non-Person Entity). Encrypted files are placed in the same directory as the input files, with a .tdf extension added to the file name. */ -func EncryptFilesGlobNPE(pattern string, config OpentdfConfig, dataAttributes []string) ([]string, error) { +func EncryptFilesWithExtensionsNPE(dirPath string, extensions []string, config OpentdfConfig, dataAttributes []string) ([]string, error) { authScopes := []string{"email"} sdkClient, err := newSdkClient(config, authScopes) if err != nil { return nil, err } - files, err := filepath.Glob(pattern) + files, err := findFiles(dirPath, extensions) if err != nil { return nil, err } - var outputPaths []string - for _, inputFilePath := range files { + var outputPaths = make([]string, 0, len(files)) + var errors = make([]error, 0, len(files)) + for _, file := range files { + inputFilePath := file outputFilePath := inputFilePath + ".tdf" got, err := encryptFileWithClient(inputFilePath, outputFilePath, sdkClient, config, dataAttributes) if err != nil { - - return nil, fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err) - } else { - outputPaths = append(outputPaths, got) + errors = append(errors, fmt.Errorf("failed to encrypt file %s: %v", inputFilePath, err)) + continue } + outputPaths = append(outputPaths, got) + } + + logOutputPaths(outputPaths, errors) + + if len(errors) > 0 { + return outputPaths, fmt.Errorf("encountered errors during encryption: %v", errors) } return outputPaths, nil } @@ -469,17 +508,7 @@ func EncryptFilePE(inputFilePath string, outputFilePath string, config OpentdfCo } /* -TODO: Create a single global var for sdkClinet (global var) - - E.g. in an HTTP server, create an instance for each connection - -TODO: The platform knows about the IdP, therefore we don't need -to specify the TOKEN_ENDPOINT. - -TODO: Research why the platform is hard-coding "email" for scope - -A non-Public decrypt function. - -Based on: +A non-Public decrypt function, based on: - https://github.com/opentdf/otdfctl/blob/46cfca1ba32c57f7264c320db27394c00412ca49/pkg/handlers/tdf.go#L29-L41 */ func decryptBytes(toDecrypt []byte, authScopes []string, config OpentdfConfig) (*bytes.Buffer, error) { @@ -555,98 +584,175 @@ in the same directory as the input files, with the .tdf extension removed from t */ func DecryptFilesInDirNPE(dirPath string, config OpentdfConfig) ([]string, error) { authScopes := []string{"email"} - sdkClient, err := newSdkClient(config, authScopes) - if err != nil { - return nil, err - } files, err := os.ReadDir(dirPath) if err != nil { return nil, err } - var outputPaths []string + var wg sync.WaitGroup + outputPathsChan := make(chan string, len(files)) + errChan := make(chan error, len(files)) + for _, file := range files { if !file.IsDir() && strings.HasSuffix(file.Name(), ".tdf") { - inputFilePath := path.Join(dirPath, file.Name()) - outputFilePath := strings.TrimSuffix(inputFilePath, ".tdf") - - bytes, err := readBytesFromFile(inputFilePath) - if err != nil { + wg.Add(1) + go func(file os.DirEntry) { + defer wg.Done() + sdkClient, err := newSdkClient(config, authScopes) + if err != nil { + errChan <- fmt.Errorf("failed to create SDK client: %v", err) + return + } + + fileInfo, err := file.Info() + if err != nil { + errChan <- fmt.Errorf("failed to get file info for %s: %v", file.Name(), err) + return + } + inputFilePath := path.Join(dirPath, fileInfo.Name()) + outputFilePath := strings.TrimSuffix(inputFilePath, ".tdf") + + bytes, err := readBytesFromFile(inputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to read file %s: %v", inputFilePath, err) + return + } + + decrypted, err := decryptBytesWithClient(bytes, sdkClient) + if err != nil { + errChan <- fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err) + return + } + + tdfFile, err := os.Create(outputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err) + return + } + defer tdfFile.Close() + + _, e := io.Copy(tdfFile, decrypted) + if e != nil { + errChan <- fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err) + return + } + + outputPathsChan <- outputFilePath + }(file) + } + } - return nil, fmt.Errorf("failed to read file %s: %v", inputFilePath, err) - } + wg.Wait() + close(outputPathsChan) + close(errChan) - decrypted, err := decryptBytesWithClient(bytes, sdkClient) - if err != nil { - return nil, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err) - } + var outputPaths []string + for path := range outputPathsChan { + outputPaths = append(outputPaths, path) + } - tdfFile, err := os.Create(outputFilePath) - if err != nil { - return nil, fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err) - } - defer tdfFile.Close() + var errors []error + for err := range errChan { + errors = append(errors, err) + } - _, e := io.Copy(tdfFile, decrypted) - if e != nil { - return nil, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err) - } + logOutputPaths(outputPaths, errors) - outputPaths = append(outputPaths, outputFilePath) - } + if len(errors) > 0 { + return nil, fmt.Errorf("encountered errors during decryption: %v", errors) } + return outputPaths, nil } /* -DecryptFilesGlobNPE decrypts all files matching the specified glob pattern. +DecryptFilesWithExtensionsNPE decrypts all files matching the file 'extensions' in 'dirPath'. Work is performed as an NPE (Non-Person Entity). Decrypted files are placed in the same directory as the input files, with the .tdf extension removed from the file name. */ -func DecryptFilesGlobNPE(pattern string, config OpentdfConfig) ([]string, error) { +func DecryptFilesWithExtensionsNPE(dirPath string, extensions []string, config OpentdfConfig) ([]string, error) { authScopes := []string{"email"} - sdkClient, err := newSdkClient(config, authScopes) - if err != nil { - return nil, err - } - files, err := filepath.Glob(pattern) + files, err := os.ReadDir(dirPath) if err != nil { return nil, err } - var outputPaths []string - for _, inputFilePath := range files { - if strings.HasSuffix(inputFilePath, ".tdf") { - outputFilePath := strings.TrimSuffix(inputFilePath, ".tdf") + outputPathsChan := make(chan string, len(files)) + errChan := make(chan error, len(files)) - bytes, err := readBytesFromFile(inputFilePath) - if err != nil { - return nil, fmt.Errorf("failed to read file %s: %v", inputFilePath, err) - } + var wg sync.WaitGroup - decrypted, err := decryptBytesWithClient(bytes, sdkClient) - if err != nil { - return nil, fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err) + for _, file := range files { + if !file.IsDir() { + for _, ext := range extensions { + if strings.HasSuffix(file.Name(), ext) { + wg.Add(1) + go func(file os.DirEntry, ext string) { + defer wg.Done() + sdkClient, err := newSdkClient(config, authScopes) + if err != nil { + errChan <- fmt.Errorf("failed to create SDK client: %v", err) + return + } + + inputFilePath := filepath.Join(dirPath, file.Name()) + outputFilePath := strings.TrimSuffix(inputFilePath, ext) + + bytes, err := readBytesFromFile(inputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to read file %s: %v", inputFilePath, err) + return + } + + decrypted, err := decryptBytesWithClient(bytes, sdkClient) + if err != nil { + errChan <- fmt.Errorf("failed to decrypt file %s: %v", inputFilePath, err) + return + } + + tdfFile, err := os.Create(outputFilePath) + if err != nil { + errChan <- fmt.Errorf("failed to write decrypted file %s: %v", outputFilePath, err) + return + } + defer tdfFile.Close() + + _, e := io.Copy(tdfFile, decrypted) + if e != nil { + errChan <- fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err) + return + } + + outputPathsChan <- outputFilePath + }(file, ext) + } } + } + } - tdfFile, err := os.Create(outputFilePath) - if err != nil { - return nil, fmt.Errorf("failed to write decrypted file %s", outputFilePath) - } - defer tdfFile.Close() + wg.Wait() + close(outputPathsChan) + close(errChan) - _, e := io.Copy(tdfFile, decrypted) - if e != nil { - return nil, fmt.Errorf("failed to write decrypted data to destination %s: %v", outputFilePath, err) - } + var outputPaths []string + for path := range outputPathsChan { + outputPaths = append(outputPaths, path) + } - outputPaths = append(outputPaths, outputFilePath) - } + var errors []error + for err := range errChan { + errors = append(errors, err) } + + logOutputPaths(outputPaths, errors) + if len(outputPaths) == 0 { - return nil, fmt.Errorf("no files were decrypted") + if len(errors) == 0 { + return nil, fmt.Errorf("no files with extensions %v found in directory %s", extensions, dirPath) + } + return nil, fmt.Errorf("encountered errors during decryption of files in directory %s: %v", dirPath, errors) } return outputPaths, nil } @@ -736,3 +842,43 @@ func encryptBytesWithClient(b []byte, sdkClient *sdk.SDK, config OpentdfConfig, } return enc, nil } + +// Function to find all files recursively in a directory matching the given extensions +func findFiles(dir string, extensions []string) ([]string, error) { + var files []string + + // Use filepath.Walk to walk through the directory recursively + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + // If there's an error reading the file, skip it + return err + } + + // Check if the file extension matches 'extensions' parameter + if !info.IsDir() && strings.Contains(strings.Join(extensions, ","), filepath.Ext(path)) { + files = append(files, path) // Add the file to the list + } + + return nil + }) + + if err != nil { + return nil, err + } + + return files, nil +} + +// logOutputPaths logs the output paths and any errors that occurred during processing +func logOutputPaths(outputPaths []string, errors []error) { + if len(errors) > 0 { + log.Println("Errors occurred during processing:") + for _, err := range errors { + log.Printf("\t%s\n", err) + } + } + log.Println("Output Paths:") + for _, path := range outputPaths { + log.Printf("\t%s\n", path) + } +} diff --git a/otdf_python_test.go b/otdf_python_test.go index 939e940..3fc835e 100644 --- a/otdf_python_test.go +++ b/otdf_python_test.go @@ -346,7 +346,7 @@ func Test_PE_E2E_File_Multi_Attributes(t *testing.T) { e2e_test_as_PE(t, attrValues) } -func Test_NPE_Encrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { +func Test_Multifile_NPE_Encrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { // Create a temporary directory tmpDir, err := os.MkdirTemp("", "input-dir") if err != nil { @@ -378,26 +378,40 @@ func Test_NPE_Encrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { t.Fatal("Unable to write to temporary file", err) } - // Call the EncryptFilesInDirNPE function - got, err := gotdf_python.EncryptFilesInDirNPE(tmpDir, gotdf_python.OpentdfConfig{ + // Create a temporary file in the directory + tmpFile3, err := os.CreateTemp(tmpDir, "input-file3-*.csv") + if err != nil { + t.Fatal("Could not create input file", err) + } + defer tmpFile3.Close() + + // Write some data to the file + if _, err = tmpFile3.WriteString("test data"); err != nil { + t.Fatal("Unable to write to temporary file", err) + } + + cfg := gotdf_python.OpentdfConfig{ ClientId: config.npeClientId, ClientSecret: config.npeClientSecret, PlatformEndpoint: config.platformEndpoint, TokenEndpoint: config.tokenEndpoint, KasUrl: config.kasEndpoint, - }, nil) + } + + got, err := gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt", ".csv"}, cfg, nil) if err != nil { - t.Fatal("Failed to EncryptFilesInDirNPE()!", err) + t.Fatal("Failed to EncryptFilesWithExtensionsNPE()!", err) } - if len(got) == 0 { - t.Fatal("EncryptFilesInDirNPE returned empty value, but didn't error!") + if len(got) != 3 { + t.Fatal("EncryptFilesWithExtensionsNPE returned incorrect got value, but didn't error!") } - fmt.Println("Successfully encrypted files in directory") + fmt.Println("Successfully encrypted files using file extensions") } -func Test_NPE_Encrypt_Files_Glob_Nil_Attributes(t *testing.T) { +// A new test of a new 'EncryptFilesWithExtensions' function +func Test_Multifile_NPE_Encrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { // Create a temporary directory tmpDir, err := os.MkdirTemp("", "input-dir") if err != nil { @@ -405,32 +419,11 @@ func Test_NPE_Encrypt_Files_Glob_Nil_Attributes(t *testing.T) { } defer os.RemoveAll(tmpDir) - // Create a temporary file in the directory - tmpFile1, err := os.CreateTemp(tmpDir, "input-file1-*.txt") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile1.Close() - - // Write some data to the file - if _, err = tmpFile1.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } - - // Create a temporary file in the directory - tmpFile2, err := os.CreateTemp(tmpDir, "input-file2-*.txt") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile2.Close() - - // Write some data to the file - if _, err = tmpFile2.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } + // Create test files + numFiles := createTestFiles(t, tmpDir) - // Call the EncryptFilesGlobNPE function - got, err := gotdf_python.EncryptFilesGlobNPE(tmpDir+"/*.txt", gotdf_python.OpentdfConfig{ + // Call the EncryptFilesWithExtensionsNPE function + got, err := gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt", ".csv", ".pdf"}, gotdf_python.OpentdfConfig{ ClientId: config.npeClientId, ClientSecret: config.npeClientSecret, PlatformEndpoint: config.platformEndpoint, @@ -438,18 +431,18 @@ func Test_NPE_Encrypt_Files_Glob_Nil_Attributes(t *testing.T) { KasUrl: config.kasEndpoint, }, nil) if err != nil { - t.Fatal("Failed to EncryptFilesGlobNPE()!", err) + t.Fatal("Failed to EncryptFilesWithExtensionsNPE()!", err) } - if len(got) == 0 { - t.Fatal("EncryptFilesGlobNPE returned empty value, but didn't error!") + if len(got) != numFiles { + t.Fatal("EncryptFilesWithExtensionsNPE returned incorrect got value, but didn't error!") } - fmt.Println("Successfully encrypted files using glob pattern") + fmt.Println("Successfully encrypted files with extensions") } // Call the DecryptFilesInDirNPE function -func Test_NPE_Decrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { +func Test_Multifile_NPE_Decrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { // Create a temporary directory tmpDir, err := os.MkdirTemp("", "input-dir") if err != nil { @@ -457,29 +450,8 @@ func Test_NPE_Decrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { } defer os.RemoveAll(tmpDir) - // Create a temporary file in the directory - tmpFile1, err := os.CreateTemp(tmpDir, "input-file1-*.txt") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile1.Close() - - // Write some data to the file - if _, err = tmpFile1.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } - - // Create a temporary file in the directory - tmpFile2, err := os.CreateTemp(tmpDir, "input-file2-*.txt") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile2.Close() - - // Write some data to the file - if _, err = tmpFile2.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } + // Create test files + numFiles := createTestFiles(t, tmpDir) // Encrypt the file _, err = gotdf_python.EncryptFilesInDirNPE(tmpDir, gotdf_python.OpentdfConfig{ @@ -505,14 +477,14 @@ func Test_NPE_Decrypt_Files_In_Dir_Nil_Attributes(t *testing.T) { t.Fatal("Failed to DecryptFilesInDirNPE()!", err) } - if len(got) == 0 { + if len(got) != numFiles { t.Fatal("DecryptFilesInDirNPE returned empty value, but didn't error!") } fmt.Println("Successfully decrypted files in directory") } -func Test_NPE_Decrypt_Files_Glob_Nil_Attributes(t *testing.T) { +func Test_Multifile_NPE_Decrypt_Files_With_Extensions_Nil_Attributes(t *testing.T) { // Create a temporary directory tmpDir, err := os.MkdirTemp("", "input-dir") if err != nil { @@ -520,32 +492,11 @@ func Test_NPE_Decrypt_Files_Glob_Nil_Attributes(t *testing.T) { } defer os.RemoveAll(tmpDir) - // Create a temporary file in the directory - tmpFile1, err := os.CreateTemp(tmpDir, "input-file1-*.txt") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile1.Close() - - // Write some data to the file - if _, err = tmpFile1.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } + // Create test files + numFiles := createTestFiles(t, tmpDir) - // Create a temporary file in the directory - tmpFile2, err := os.CreateTemp(tmpDir, "input-file2-*.txt") - if err != nil { - t.Fatal("Could not create input file", err) - } - defer tmpFile2.Close() - - // Write some data to the file - if _, err = tmpFile2.WriteString("test data"); err != nil { - t.Fatal("Unable to write to temporary file", err) - } - - // Encrypt the file - _, err = gotdf_python.EncryptFilesGlobNPE(tmpDir+"/*.txt", gotdf_python.OpentdfConfig{ + // Encrypt the files + _, err = gotdf_python.EncryptFilesWithExtensionsNPE(tmpDir, []string{".txt", ".csv", ".pdf"}, gotdf_python.OpentdfConfig{ ClientId: config.npeClientId, ClientSecret: config.npeClientSecret, PlatformEndpoint: config.platformEndpoint, @@ -553,11 +504,11 @@ func Test_NPE_Decrypt_Files_Glob_Nil_Attributes(t *testing.T) { KasUrl: config.kasEndpoint, }, nil) if err != nil { - t.Fatal("Failed to EncryptFilesGlobNPE()!", err) + t.Fatal("Failed to EncryptFilesWithExtensionsNPE()!", err) } - // Call the DecryptFilesGlobNPE function - got, err := gotdf_python.DecryptFilesGlobNPE(tmpDir+"/*.tdf", gotdf_python.OpentdfConfig{ + // Call the DecryptFilesWithExtensionsNPE function + got, err := gotdf_python.DecryptFilesWithExtensionsNPE(tmpDir, []string{".tdf"}, gotdf_python.OpentdfConfig{ ClientId: config.npeClientId, ClientSecret: config.npeClientSecret, PlatformEndpoint: config.platformEndpoint, @@ -565,12 +516,43 @@ func Test_NPE_Decrypt_Files_Glob_Nil_Attributes(t *testing.T) { KasUrl: config.kasEndpoint, }) if err != nil { - t.Fatal("Failed to DecryptFilesGlobNPE()!", err) + t.Fatal("Failed to DecryptFilesWithExtensionsNPE()!", err) } - if len(got) == 0 { - t.Fatal("DecryptFilesGlobNPE returned empty value, but didn't error!") + if len(got) != numFiles { + t.Fatal("DecryptFilesWithExtensionsNPE returned empty value, but didn't error!") + } + + fmt.Println("Successfully decrypted files with extensions") +} + +func createTestFiles(t *testing.T, tmpDir string) int { + // A number that corresponds to the hour of the day (between 0 and 23) + numFiles := time.Now().Hour() + + if numFiles > 12 { + numFiles = numFiles - 12 // Limit the number of files to 12 + } + + for i := 0; i < numFiles; i++ { + ext := ".txt" + if i%2 == 0 { + ext = ".csv" + } else if i%3 == 0 { + ext = ".pdf" + } + + tmpFile, err := os.CreateTemp(tmpDir, fmt.Sprintf("input-file-%d-*%s", i, ext)) + if err != nil { + t.Fatal("Could not create input file", err) + } + defer tmpFile.Close() + + // Write some data to the file + if _, err = tmpFile.WriteString("test data"); err != nil { + t.Fatal("Unable to write to temporary file", err) + } } - fmt.Println("Successfully decrypted files using glob pattern") + return numFiles } diff --git a/pyproject.toml b/pyproject.toml index 77e5e7e..3081133 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "otdf-python" # Should match 'setup.py' version number (used for gopy/pybindgen) -version = "0.2.4" +version = "0.2.9" description = "Unofficial OpenTDF SDK for Python." authors = [ {name="b-long", email="b-long@users.noreply.github.com"} @@ -19,7 +19,7 @@ pybindgen = "^0.22.1" [tool.poetry] package-mode = false -version = "0.2.4" +version = "0.2.9" [tool.poetry.dependencies] python = ">=3.11,<3.14" diff --git a/setup.py b/setup.py index ad39f39..a140562 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url="https://github.com/b-long/opentdf-python-sdk", package_data={"otdf_python": ["*.so"]}, # Should match 'pyproject.toml' version number - version="0.2.4", + version="0.2.9", author_email="b-long@users.noreply.github.com", include_package_data=True, ) diff --git a/setup_ci.py b/setup_ci.py index 5dddaa2..bfac080 100644 --- a/setup_ci.py +++ b/setup_ci.py @@ -81,7 +81,7 @@ def build_extension(self, ext: Extension): setuptools.setup( name="otdf_python", - version="0.2.4", + version="0.2.9", author="b-long", description="Unofficial OpenTDF SDK for Python.", long_description_content_type="text/markdown", diff --git a/uv.lock b/uv.lock index 4c21e9b..59938cf 100644 --- a/uv.lock +++ b/uv.lock @@ -3,5 +3,5 @@ requires-python = ">=3.11" [[package]] name = "otdf-python" -version = "0.2.4" +version = "0.2.9" source = { editable = "." }