diff --git a/powershell/VismaDeveloperPortal/upload/README.md b/powershell/VismaDeveloperPortal/upload/README.md index 461dd71..3e630a2 100644 --- a/powershell/VismaDeveloperPortal/upload/README.md +++ b/powershell/VismaDeveloperPortal/upload/README.md @@ -5,7 +5,8 @@ A collection of examples to upload files with the File API using **PowerShell**. ## Prerequisites - The script was prepared for PowerShell version 5.1 or above. With lower versions it might not work properly. -- Files to be uploaded cannot be bigger than 100 MB. +- Files to be uploaded cannot be bigger than 10 GB. +- Chunksize is limited to 100 MB (4 MB is suggested as most efficient ## Getting Started @@ -131,6 +132,13 @@ Inside the **config.xml** file you will find these parameters:
+**`ChunkSize`** +> Size of chunks (MB) used to send large files (default: 4 MB is most efficient). +> +> **Example:** 4 + +
+ ## Example of a valid configuration ```xml @@ -153,6 +161,7 @@ Inside the **config.xml** file you will find these parameters: C:\Visma\File API\Ftaas.Examples\powershell\VismaDeveloperPortal\upload data*.xml C:\Visma\File API\Ftaas.Examples\powershell\VismaDeveloperPortal\archive + 4 ``` diff --git a/powershell/VismaDeveloperPortal/upload/UploadFile.ps1 b/powershell/VismaDeveloperPortal/upload/UploadFile.ps1 index ed0e611..a710d11 100644 --- a/powershell/VismaDeveloperPortal/upload/UploadFile.ps1 +++ b/powershell/VismaDeveloperPortal/upload/UploadFile.ps1 @@ -1,4 +1,4 @@ -# This example shows how to upload a file. +# This example shows how to upload a file. # Authors: Visma - Transporters Team [CmdletBinding()] @@ -26,7 +26,8 @@ if (-not $_configPath) { } Write-Host "=========================================================" -Write-Host "File API example: Upload a file." +Write-Host "File API example: Upload files from a directory." +Write-Host " Supports files > 100Mb" Write-Host "=========================================================" Write-Host "(you can stop the script at any moment by pressing the buttons 'CTRL'+'C')" @@ -78,7 +79,7 @@ catch { #endregion Retrieve authentication token $fileApiClient = [FileApiClient]::new($config.Services.FileApiBaseUrl, $token) -$fileApiService = [FileApiService]::new($fileApiClient, $config.Upload.BusinessTypeId) +$fileApiService = [FileApiService]::new($fileApiClient, $config.Upload.BusinessTypeId, $config.Upload.ChunkSize) #region Upload Directory contents @@ -87,8 +88,7 @@ Get-ChildItem -Path $config.Upload.Path -Filter $config.Upload.Filter | ForEach- $filenameToUpload = $_.FullName try { - $createdFilePath = $fileApiService.CreateFileToUpload($filenameToUpload) - $fileApiService.UploadFile($createdFilePath, $(Split-Path -Path $_.FullName -Leaf)) + $fileApiService.UploadFile($filenameToUpload) } catch { [Helper]::EndProgramWithError($_, "Failure uploading file $($filenameToUpload).") @@ -100,9 +100,10 @@ Get-ChildItem -Path $config.Upload.Path -Filter $config.Upload.Filter | ForEach- catch { [Helper]::EndProgramWithError($_, "Failure archiving file to $($archivedFile).") } - } + + #endregion Upload Directory contents [Helper]::EndProgram() @@ -133,6 +134,8 @@ class ConfigurationManager { $businessTypeId = $config.Upload.BusinessTypeId $contentDirectoryPath = $config.Upload.Path $contentFilter = $config.Upload.Filter + $chunkSize = [long] $config.Upload.ChunkSize + $chunkSizeLimit = [long] 100 $archivePath = $config.Upload.ArchivePath @@ -145,7 +148,8 @@ class ConfigurationManager { if ([string]::IsNullOrEmpty($contentDirectoryPath)) { $missingConfiguration += "Upload.Path" } if ([string]::IsNullOrEmpty($contentFilter)) { $missingConfiguration += "Upload.Filter" } if ([string]::IsNullOrEmpty($archivePath)) { $missingConfiguration += "Upload.ArchivePath" } - + if ([string]::IsNullOrEmpty($chunkSize)) { $missingConfiguration += "Upload.ChunkSize" } + if ($missingConfiguration.Count -gt 0) { throw "Missing parameters: $($missingConfiguration -Join ", ")" } @@ -156,6 +160,9 @@ class ConfigurationManager { if (-not [Validator]::IsUri($authenticationTokenApiBaseUrl)) { $wrongConfiguration += "Services.AuthenticationTokenApiBaseUrl" } if (-not [Validator]::IsPath($contentDirectoryPath)) { $wrongConfiguration += "Upload.Path" } if (-not [Validator]::IsPath($archivePath)) { $wrongConfiguration += "Upload.ArchivePath" } + if($chunkSize -gt $chunkSizeLimit) { $wrongConfiguration += "Chunk size ($($chunkSize)) cannot be bigger than $($chunkSizeLimit) bytes."} + if($chunkSize -lt 1){$wrongConfiguration += "Chunk size ($($chunkSize)) cannot be smaller than 1 (Mbyte)."} + if ($wrongConfiguration.Count -gt 0) { throw "Wrong configured parameters: $($wrongConfiguration -Join ", ")" @@ -170,6 +177,7 @@ class ConfigurationManager { $configuration.Upload.Path = $contentDirectoryPath $configuration.Upload.Filter = $contentFilter $configuration.Upload.ArchivePath = $archivePath + $configuration.Upload.ChunkSize = $chunkSize * 1024 * 1024 #convert to bytes Write-Host "Configuration retrieved." @@ -267,29 +275,74 @@ class CredentialsManager { class FileApiService { hidden [FileApiClient] $_fileApiClient - hidden [long] $_uploadSizeLimit + hidden [long] $_fileSize + hidden [long] $_fileBytesRead hidden [string] $_boundary hidden [string] $_businessTypeId + hidden [long] $_chunkSize + hidden [int] $_uploadDelay FileApiService( [FileApiClient] $fileApiClient, - [string] $businessTypeId + [string] $businessTypeId, + [long] $chunkSize ) { $this._fileApiClient = $fileApiClient $this._boundary = "file_info" $this._businessTypeId = $businessTypeId + $this._chunkSize = $chunkSize + $this._uploadDelay = 0 + } + + [void] UploadFile($filenameToUpload){ + Write-Host "----" + Write-Host "Uploading the file." + Write-Host "| File: $($(Split-Path -Path $filenameToUpload -Leaf))" + Write-Host "| Business type: $($this._businessTypeId)" + + $result = $this.UploadFirstRequest($filenameToUpload) + $fileToken = $result.FileToken + $chunkNumber = 1 + While( -not $result.Eof ) { + Write-Host "Uploading Chunk #$($chunkNumber + 1)." + $result = $this.UploadChunkRequest($fileToken, $filenameToUpload, $chunkNumber) + $chunkNumber += 1 + } + Write-Host "File $($(Split-Path -Path $filenameToUpload -Leaf)) uploaded." + } + + [PSCustomObject] UploadFirstRequest([string] $filenameToUpload){ + $this._fileSize = (Get-Item $filenameToUpload).Length + $this._fileBytesRead = 0 + $filenameOnly = Split-Path -Path $filenameToUpload -Leaf + $chunkNumber = 0 + $result = $this.CreateChunk($filenameToUpload, $this._chunkSize, $chunkNumber) + $FirstRequestData = $this.CreateFirstRequestToUpload($filenameOnly, $result.ChunkPath) + $response = $this.UploadFile($FirstRequestData, $filenameOnly, "multipart/related;boundary=$($this._boundary)", "", $chunkNumber, $result.Eof) + $fileToken = $response.uploadToken + + return [PSCustomObject]@{ + FileToken = $fileToken + Eof = $result.Eof + } + } + + [PSCustomObject] UploadChunkRequest([string] $fileToken, [string] $filenameToUpload, [long] $chunkNumber){ + $result = $this.CreateChunk($filenameToUpload, $this._chunkSize, $chunkNumber) + $response = $this.UploadFile($result.ChunkPath, $(Split-Path -Path $filenameToUpload -Leaf), "application/octet-stream", $fileToken, $chunkNumber, $result.Eof) - # API supports files up to 100 megabytes - $this._uploadSizeLimit = 100 * 1024 * 1024 + return [PSCustomObject]@{ + Eof = $result.Eof + } } - [string] CreateFileToUpload([string] $filename) { + [string] CreateFirstRequestToUpload([string] $filename, [string] $chunkPath) { Write-Host "----" - Write-Host "Creating a bundle with the file $($filename) to upload." + Write-Host "Creating first request with the file $($filename) to upload." $headerFilePath = "" $footerFilePath = "" try { - $folderPath = $(Split-Path -Path $filename) + $folderPath = $(Split-Path -Path $chunkPath) $contentFilename = $(Split-Path -Path $filename -Leaf) $createdFilePath = "$($folderPath)\$([Helper]::ConvertToUniqueFilename("multipart.bin"))" @@ -308,7 +361,7 @@ class FileApiService { New-Item -Path $folderPath -Name $headerFilename -Value $headerContent New-Item -Path $folderPath -Name $footerFilename -Value $footerContent - cmd /c copy /b $headerFilePath + $filename + $footerFilePath $createdFilePath + cmd /c copy /b $headerFilePath + $chunkPath + $footerFilePath $createdFilePath Write-Host "File created." return $createdFilePath } @@ -319,21 +372,67 @@ class FileApiService { if (Test-Path $footerFilePath) { Remove-Item -Force -Path $footerFilePath } + if (Test-Path $chunkPath) { + Remove-Item -Force -Path $chunkPath + } } } - [void] UploadFile([string] $filePath, $originalFilename) { - if ((Get-Item $filePath).Length -gt $this._uploadSizeLimit) { - throw "Cannot upload files bigger $($this._uploadSizeLimit) bytes." + [PSCustomObject] CreateChunk([string] $contentFilePath, [long] $chunkSize, [long] $chunkNumber){ + $folderPath = $(Split-Path -Path $contentFilePath) + $contentFilename = $(Split-Path -Path $contentFilePath -Leaf) + $createdChunkPath = "$($folderPath)\$([Helper]::ConvertToUniqueFilename("Chunk_$($chunkNumber).bin"))" + [byte[]]$bytes = new-object Byte[] $chunkSize + $fileStream = New-Object System.IO.FileStream($contentFilePath, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read) + $binaryReader = New-Object System.IO.BinaryReader( $fileStream) + $pos = $binaryReader.BaseStream.Seek($chunkNumber * $chunkSize, [System.IO.SeekOrigin]::Begin) + $bytes = $binaryReader.ReadBytes($chunkSize) + + $this._fileBytesRead += $bytes.Length + [bool] $streamEof = 0 + if(($bytes.Length -lt $chunkSize) -or ($this._fileBytesRead -ge $this._fileSize)) { + $streamEof = 1 } - Write-Host "----" - Write-Host "Uploading the file." - Write-Host "| File: $($originalFilename)" - Write-Host "| Business type: $($this._businessTypeId)" - $this._fileApiClient.UploadFile($filePath, $this._boundary) - - Write-Host "File uploaded." + + $binaryReader.Dispose() + + Set-Content -Path $createdChunkPath -Value $bytes -Encoding Byte + + return [PSCustomObject]@{ + ChunkPath = $createdChunkPath + Eof = $streamEof + } } + + [PSCustomObject] UploadFile([string] $filePath, [string] $originalFilename, [string] $contentType, [string] $token, [long] $chunkNumber, [bool] $close) { + + while(1 -eq 1) { + try{ + Start-Sleep -Milliseconds $this._uploadDelay + + $result = $this._fileApiClient.UploadFile($filePath, $contentType, $token, $chunkNumber, $close ) + + if (-not [string]::IsNullOrEmpty($filePath) -and (Test-Path $filePath)) { + Remove-Item -Force -Path $filePath + } + + return $result + } + catch{ + if( $_.Exception.Message.Contains("(429)")){ + $this._uploadDelay += 100 + Write-Host "Spike arrest detected: Setting uploadDelay to $($this._uploadDelay) msec." + Write-Host "Waiting 60 seconds for spike arrest to clear" + Start-Sleep -Seconds 60 + } + else { + throw "$($_)" + } + } + } + + throw "UploadFile aborted: should never come here" + } } class FileApiClient { @@ -351,24 +450,43 @@ class FileApiClient { } } - [PSCustomObject] UploadFile([string] $multipartcontentFilePath, [string] $boundary) { + [PSCustomObject] UploadFile([string] $bodyPath, [string] $contentType, [string] $token, [long] $chunkNumber, [bool] $close) { $headers = $this._defaultHeaders - $headers["Content-Type"] = "multipart/related;boundary=$($boundary)" - try { + if(-not [string]::IsNullOrEmpty($contentType)){ + $headers["Content-Type"] = $contentType + } + $uri = "$($this.BaseUrl)/files" + if(($chunkNumber -eq 0) -and $close) { + $uri += "?uploadType=multipart" + } + else { + $uri += "?uploadType=resumable" + } + if(-not [string]::IsNullOrEmpty($token)){ + $uri += "&uploadToken=$($token)" + } + if($chunkNumber -ne 0){ + $uri += "&position=$($chunkNumber)" + } + if($close -and ($chunkNumber -gt 0)) { + $uri += "&close=true" + } + if($chunkNumber -eq 0){ $response = Invoke-RestMethod ` -Method "Post" ` - -Uri "$($this.BaseUrl)/files?uploadType=multipart" ` + -Uri $uri ` -Headers $headers ` - -InFile "$($multipartcontentFilePath)" - - return $response - } - - finally { - if (Test-Path $multipartcontentFilePath) { - Remove-Item -Force -Path $multipartcontentFilePath + -InFile "$($bodyPath)" } - } + else { + $response = Invoke-RestMethod ` + -Method "Put" ` + -Uri $uri ` + -Headers $headers ` + -InFile "$($bodyPath)" + } + + return $response } } @@ -583,6 +701,7 @@ class ConfigurationSectionUpload { [string] $Path [string] $Filter [string] $ArchivePath + [long] $ChunkSize } class FileInfo { diff --git a/powershell/VismaDeveloperPortal/upload/config.xml b/powershell/VismaDeveloperPortal/upload/config.xml index 17f6881..23bf626 100644 --- a/powershell/VismaDeveloperPortal/upload/config.xml +++ b/powershell/VismaDeveloperPortal/upload/config.xml @@ -17,5 +17,6 @@ EnterTheDirectoryToUpload (e.g.: C:\Visma\File API\Ftaas.Examples\powershell\upload\VismaDeveloperPortal\file) EnterFilemaskToUseForUpload (e.g.: *.xml) EnterTheDirectoryForArchivedFiles (e.g.: C:\Visma\File API\Ftaas.Examples\powershell\VismaDeveloperPortal\archive) + 4 diff --git a/powershell/YouforceDeveloperPortal/download/DownloadFiles.ps1 b/powershell/YouforceDeveloperPortal/download/DownloadFiles.ps1 deleted file mode 100644 index 5726c34..0000000 --- a/powershell/YouforceDeveloperPortal/download/DownloadFiles.ps1 +++ /dev/null @@ -1,605 +0,0 @@ -# This example shows how to download all the files specified in a filter. -# Authors: Visma - Transporters Team - -[CmdletBinding()] -Param( - [Alias("ConfigPath")] - [Parameter( - Mandatory = $false, - HelpMessage = 'Full path of the configuration (e.g. C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\download\config.xml). Default value: set in the code.' - )] - [string] $_configPath, - - [Alias("RenewCredentials")] - [Parameter( - Mandatory = $false, - HelpMessage = 'Boolean. $true if you want to renew your credentials. $false otherwise' - )] - [bool] $_renewCredentials = $false -) - -$ErrorActionPreference = "Stop" - -# The default value of this parameter is set here because $PSScriptRoot is empty if used directly in Param() through PowerShell ISE. -if (-not $_configPath) { - $_configPath = "$($PSScriptRoot)\config.xml" -} - -Write-Host "=========================================================" -Write-Host "File API example: Download files specified in a filter." -Write-Host "=========================================================" - -Write-Host "(you can stop the script at any moment by pressing the buttons 'CTRL'+'C')" - -#region Configuration - -[ConfigurationManager] $configurationManager = [ConfigurationManager]::new() - -try { - $config = $configurationManager.Get($_configPath) -} -catch { - [Helper]::EndProgramWithError($_, "Failure retrieving the configuration. Tip: see the README.MD to check the format of the parameters.") -} - -#endregion Configuration - -#region Retrieve/Create credentials - -$credentialsManager = [CredentialsManager]::new($config.Credentials.Path) -$credentialsService = [CredentialsService]::new($credentialsManager) - -try { - if ($_renewCredentials) { - $credentials = $credentialsService.CreateNew() - } - else { - $credentials = $credentialsService.Retrieve() - } -} -catch { - [Helper]::EndProgramWithError($_, "Failure retrieving the credentials.") -} - -#endregion Retrieve/Create credentials - -#region Retrieve authentication token - -$authenticationApiClient = [AuthenticationApiClient]::new($config.Services.AuthenticationTokenApiBaseUrl) -$authenticationApiService = [AuthenticationApiService]::new($authenticationApiClient) - -try { - $token = $authenticationApiService.NewToken($credentials.ClientId, $credentials.ClientSecret) -} -catch { - [Helper]::EndProgramWithError($_, "Failure retrieving the authentication token.") -} - -#endregion Retrieve authentication token - -$fileApiClient = [FileApiClient]::new($config.Services.FileApiBaseUrl, $token) -$fileApiService = [FileApiService]::new($fileApiClient, $config.Download.TenantId, $config.Download.Role, 200) - -#region List files - -try { - $filesInfo = $fileApiService.GetFilesInfo($config.Download.Filter) -} -catch { - [Helper]::EndProgramWithError($_, "Failure retrieving the files.") -} - -if ($filesInfo.Count -eq 0) { - [Helper]::EndProgram() -} - -#endregion List files - -#region Download files - -try { - $fileApiService.DownloadFiles($filesInfo, $config.Download.Path, $config.Download.EnsureUniqueNames) -} -catch { - [Helper]::EndProgramWithError($_, "Failure downloading the files.") -} - -#endregion Download files - -[Helper]::EndProgram() - -# -------- END OF THE PROGRAM -------- -# Below there are classes and models to help the readability of the program - -#region Helper classes - -class ConfigurationManager { - [Configuration] Get($configPath) { - Write-Host "----" - Write-Host "Retrieving the configuration." - - if (-not (Test-Path $configPath -PathType Leaf)) { - throw "Configuration not found.`r`n| Path: $($configPath)" - } - - $configDocument = [xml](Get-Content $configPath) - $config = $configDocument.Configuration - - $credentialsPath = $config.Credentials.Path - - $fileApiBaseUrl = $config.Services.FileApiBaseUrl - $authenticationTokenApiBaseUrl = $config.Services.AuthenticationTokenApiBaseUrl - - $tenantId = $config.Download.TenantId - $role = $Config.Download.Role - $downloadPath = $config.Download.Path - $ensureUniqueNames = $config.Download.EnsureUniqueNames - $filter = $config.Download.Filter - - $missingConfiguration = @() - if ([string]::IsNullOrEmpty($credentialsPath)) { $missingConfiguration += "Credentials.Path" } - if ([string]::IsNullOrEmpty($fileApiBaseUrl)) { $missingConfiguration += "Services.FileApiBaseUrl" } - if ([string]::IsNullOrEmpty($authenticationTokenApiBaseUrl)) { $missingConfiguration += "Services.AuthenticationTokenApiBaseUrl" } - if ([string]::IsNullOrEmpty($tenantId)) { $missingConfiguration += "Download.TenantId" } - if ([string]::IsNullOrEmpty($role)) { $missingConfiguration += "Download.Role" } - if ([string]::IsNullOrEmpty($downloadPath)) { $missingConfiguration += "Download.Path" } - if ([string]::IsNullOrEmpty($ensureUniqueNames)) { $missingConfiguration += "Download.EnsureUniqueNames" } - if ($null -eq $filter) { $missingConfiguration += "Download.Filter" } - - if ($missingConfiguration.Count -gt 0) { - throw "Missing parameters: $($missingConfiguration -Join ", ")" - } - - $wrongConfiguration = @() - if (-not [Validator]::IsPath($credentialsPath)) { $wrongConfiguration += "Credentials.Path" } - if (-not [Validator]::IsUri($fileApiBaseUrl)) { $wrongConfiguration += "Services.FileApiBaseUrl" } - if (-not [Validator]::IsUri($authenticationTokenApiBaseUrl)) { $wrongConfiguration += "Services.AuthenticationTokenApiBaseUrl" } - if (-not [Validator]::IsPath($downloadPath)) { $wrongConfiguration += "Download.Path" } - if (-not [Validator]::IsBool($ensureUniqueNames)) { $wrongConfiguration += "Download.EnsureUniqueNames" } - - if ($wrongConfiguration.Count -gt 0) { - throw "Wrong configured parameters: $($wrongConfiguration -Join ", ")" - } - - $configuration = [Configuration]::new() - $configuration.Credentials.Path = $credentialsPath - $configuration.Services.FileApiBaseUrl = $fileApiBaseUrl - $configuration.Services.AuthenticationTokenApiBaseUrl = $authenticationTokenApiBaseUrl - $configuration.Download.TenantId = $tenantId - $configuration.Download.Role = $role - $configuration.Download.Path = $downloadPath - $configuration.Download.EnsureUniqueNames = [System.Convert]::ToBoolean($ensureUniqueNames) - $configuration.Download.Filter = $filter - - Write-Host "Configuration retrieved." - - return $configuration - } -} - -class CredentialsService { - hidden [CredentialsManager] $_credentialsManager - - CredentialsService ([CredentialsManager] $manager) { - $this._credentialsManager = $manager - } - - [Credentials] Retrieve() { - $credentials = $this._credentialsManager.Retrieve() - if ($null -eq $credentials) { - $credentials = $this.CreateNew() - return $credentials - } - - return $credentials - } - - [Credentials] CreateNew() { - $this._credentialsManager.CreateNew() - $credentials = $this._credentialsManager.Retrieve() - - return $credentials - } -} - -class CredentialsManager { - hidden [string] $_credentialsPath - - CredentialsManager([string] $storagePath) { - $this._credentialsPath = $storagePath - } - - [void] CreateNew() { - $storagePath = Split-Path $this._credentialsPath - - Write-Host "----" - Write-Host "Saving your credentials." - Write-Host "| Path: $($this._credentialsPath)" - - if (-not (Test-Path -Path $storagePath -PathType Container)) { - Write-Host "----" - Write-Host "Storage credential path doesn't exist. Creating it." - Write-Host "| Path: $($storagePath)" - - New-Item -ItemType Directory -Force -Path $storagePath - } - - Write-Host "Enter your credentials." - $clientId = Read-Host -Prompt '| Client ID' - $clientSecret = Read-Host -Prompt '| Client secret' -AsSecureString - - [PSCredential]::new($clientId, $clientSecret) | Export-CliXml -Path $this._credentialsPath - - Write-Host "----" - Write-Host "Credentials saved." - } - - [Credentials] Retrieve() { - Write-Host "----" - Write-Host "Retrieving your credentials." - Write-Host "| Path: $($this._credentialsPath)" - - if (-not (Test-Path -Path $this._credentialsPath -PathType Leaf)) { - Write-Host "----" - Write-Host "Credentials not found." - Write-Host "| Path: $($this._credentialsPath)" - - return $null - } - - $credentialsStorage = Import-CliXml -Path $this._credentialsPath - - $credentials = [Credentials]::new() - $credentials.ClientId = $credentialsStorage.GetNetworkCredential().UserName - $credentials.ClientSecret = $credentialsStorage.GetNetworkCredential().Password - - Write-Host "Credentials retrieved." - - return $credentials - } -} - -class FileApiService { - hidden [FileApiClient] $_fileApiClient - hidden [string] $_tenantId - hidden [string] $_role - hidden [string] $_waitTimeBetweenCallsMS - hidden [long] $_downloadSizeLimit - - FileApiService( - [FileApiClient] $fileApiClient, - [string] $tenantId, - [string] $role, - [int] $waitTimeBetweenCallsMS - ) { - $this._fileApiClient = $fileApiClient - $this._tenantId = $tenantId - $this._role = $role - $this._waitTimeBetweenCallsMS = $waitTimeBetweenCallsMS - - # This limit is set because the method Invoke-RestMethod doesn't allow - # the download of files bigger than 2 GiB. - # I set the limit a bit less than 2 GiB to give some margin. - $this._downloadSizeLimit = 2147000000 - } - - [FileInfo[]] GetFilesInfo([string] $filter) { - Write-Host "----" - Write-Host "Retrieving list of files." - if ($filter) { - Write-Host "| Tenant: $($this._tenantId)" - Write-Host "| Filter: $($filter)" - } - - $pageIndex = 0 - $pageSize = 20 - $isLastPage = $false - $filesInfo = @() - do { - $response = $this._fileApiClient.ListFiles($this._tenantId, $this._role, $pageIndex, $pageSize, $filter) - - foreach ($fileData in $response.data) { - $fileInfo = [FileInfo]::new() - $fileInfo.Id = $fileData.fileId - $fileInfo.Name = $fileData.fileName - $fileInfo.Size = $fileData.fileSize - - $filesInfo += $fileInfo - } - - $isLastPage = $pageSize * ($pageIndex + 1) -ge $response.count - $pageIndex++ - - Start-Sleep -Milliseconds $this._waitTimeBetweenCallsMS - } while (-not $isLastPage) - - Write-Host "$($filesInfo.Count) files retrieved." - - return $filesInfo - } - - [void] DownloadFiles([FileInfo[]] $filesInfo, [string] $path, [bool] $ensureUniqueNames) { - if (-not (Test-Path $path -PathType Container)) { - Write-Host "----" - Write-Host "Download path doesn't exist. Creating it." - Write-Host "| Path: $($path)" - - New-Item -ItemType Directory -Force -Path $path - } - - $downloadedFilesCount = 0 - foreach ($fileInfo in $filesInfo) { - Write-Host "----" - Write-Host "Downloading file $($downloadedFilesCount + 1)/$($filesInfo.Count)." - Write-Host "| ID: $($fileInfo.Id)" - Write-Host "| Name: $($fileInfo.Name)" - Write-Host "| Size: $($fileInfo.Size)" - - if ($fileInfo.Size -ge $this._downloadSizeLimit) { - Write-Host "----" -ForegroundColor "Red" - Write-Host "Cannot download files bigger or equal than $($this._downloadSizeLimit) bytes." -ForegroundColor "Red" - Write-Host "File will be skipped." -ForegroundColor "Red" - - continue - } - - if (($ensureUniqueNames -eq $true) -and (Test-Path "$($path)\$($fileInfo.Name)" -PathType Leaf)) { - Write-Host "There is already a file with the same name in the download path." - - $fileInfo.Name = [Helper]::ConverToUniqueFileName($fileInfo.Name) - - Write-Host "| New name: $($fileInfo.Name)" - } - - $this._fileApiClient.DownloadFile($this._tenantId, $this._role, $fileInfo, $path) - $downloadedFilesCount++ - - Write-Host "The file was downloaded." - - Start-Sleep -Milliseconds $this._waitTimeBetweenCallsMS - } - - Write-Host "----" - Write-Host "All files were downloaded." - Write-Host "| Amount: $($downloadedFilesCount)" - Write-Host "| Path: $($path)" - } -} - -class FileApiClient { - [string] $BaseUrl - - hidden [PSCustomObject] $_defaultHeaders - - FileApiClient ( - [string] $baseUrl, - [string] $token - ) { - $this.BaseUrl = $baseUrl - $this._defaultHeaders = @{ - "Authorization" = "Bearer $($token)"; - } - } - - [PSCustomObject] ListFiles([string] $tenantId, [string] $role, [int] $pageIndex, [int] $pageSize, [string] $filter) { - $headers = $this._defaultHeaders - $headers["x-raet-tenant-id"] = $tenantId - - $response = Invoke-RestMethod ` - -Method "Get" ` - -Uri "$($this.BaseUrl)/files?role=$($role)&pageIndex=$($pageIndex)&pageSize=$($pageSize)&`$filter=$($filter)&`$orderBy=uploadDate asc" ` - -Headers $headers - - return $response - } - - [PSCustomObject] DownloadFile([string] $tenantId, [string] $role, [FileInfo] $fileInfo, [string] $downloadPath) { - $headers = $this._defaultHeaders - $headers["x-raet-tenant-id"] = $tenantId - $headers.Accept = "application/octet-stream" - - $response = Invoke-RestMethod ` - -Method "Get" ` - -Uri "$($this.BaseUrl)/files/$($fileInfo.Id)?role=$($role)" ` - -Headers $headers ` - -OutFile "$($downloadPath)\$($fileInfo.Name)" - - return $response - } -} - -class AuthenticationApiService { - hidden [AuthenticationApiClient] $_authenticationApiClient - - AuthenticationApiService([AuthenticationApiClient] $authenticationApiClient) { - $this._authenticationApiClient = $authenticationApiClient - } - - [string] NewToken([string] $clientId, [string] $clientSecret) { - Write-Host "----" - Write-Host "Retrieving the authentication token." - - $response = $this._authenticationApiClient.NewToken($clientId, $clientSecret) - $token = $response.access_token - - Write-Host "Authentication token retrieved." - - return $token - } -} - -class AuthenticationApiClient { - hidden [string] $_baseUrl - - AuthenticationApiClient([string] $baseUrl) { - $this._baseUrl = $baseUrl - } - - [PSCustomObject] NewToken([string] $clientId, [string] $clientSecret) { - $headers = @{ - "Content-Type" = "application/x-www-form-urlencoded"; - "Cache-Control" = "no-cache"; - } - $body = @{ - "grant_type" = "client_credentials"; - "client_id" = $clientId; - "client_secret" = $clientSecret; - } - - $response = Invoke-RestMethod ` - -Method "Post" ` - -Uri "$($this._baseUrl)/token" ` - -Headers $headers ` - -Body $body - - return $response - } -} - -class Validator { - static [bool] IsUri([string] $testParameter) { - try { - $uri = $testParameter -As [System.Uri] - - if ($uri.AbsoluteUri) { - return $true - } - else { - return $false - } - } - catch { - return $false - } - } - - static [bool] IsBool([string] $testParameter) { - try { - $result = [bool]::TryParse($testParameter, [ref] $null) - return $result - } - catch { - return $false - } - } - - static [bool] IsPath([string] $testParameter) { - try { - $result = Test-Path $testParameter -IsValid - return $result - } - catch { - return $false - } - } -} - -class Helper { - static [string] ConverToUniqueFileName([string] $fileName) { - $fileNameInfo = [Helper]::GetFileNameInfo($fileName) - $fileNameWithoutExtension = $fileNameInfo.Name - $fileExtension = $fileNameInfo.Extension - $timestamp = Get-Date -Format FileDateTimeUniversal - - $uniqueFileName = "$($fileNameWithoutExtension)_$($timestamp)$($fileExtension)" - return $uniqueFileName - } - - static [FileNameInfo] GetFileNameInfo([string] $fileName) { - $fileNameInfo = [FileNameInfo]::new() - $fileNameInfo.Name = $fileName - $fileNameInfo.Extension = "" - - $splitFileName = $fileName -split "\." - if ($splitFileName.Length -gt 1) { - $fileNameInfo.Name = $splitFileName[0..($splitFileName.Length - 2)] -Join "." - $fileNameInfo.Extension = ".$($splitFileName[-1])" - } - - return $fileNameInfo - } - - static [void] EndProgram() { - [Helper]::FinishProgram($false) - } - - static [void] EndProgramWithError([System.Management.Automation.ErrorRecord] $errorRecord, [string] $genericErrorMessage) { - Write-Host "ERROR - $($genericErrorMessage)" -ForegroundColor "Red" - - $errorMessage = "Unknown error." - if ($errorRecord.ErrorDetails.Message) { - $errorDetails = $errorRecord.ErrorDetails.Message | ConvertFrom-Json - if ($errorDetails.message) { - $errorMessage = $errorDetails.message - } - elseif ($errorDetails.error.message) { - $errorMessage = $errorDetails.error.message - } - } - elseif ($errorRecord.Exception.message) { - $errorMessage = $errorRecord.Exception.message - } - - Write-Host "| Error message: $($errorMessage)" -ForegroundColor "Red" - Write-Host "| Error line in the script: $($errorRecord.InvocationInfo.ScriptLineNumber)" -ForegroundColor "Red" - - [Helper]::FinishProgram($true) - } - - hidden static [void] FinishProgram([bool] $finishWithError) { - Write-Host "----" - Write-Host "End of the example." - - if ($finishWithError) { - exit 1 - } - else { - exit - } - } -} - -#endregion Helper classes - -#region Models - -class Configuration { - $Credentials = [ConfigurationSectionCredentials]::new() - $Services = [ConfigurationSectionServices]::new() - $Download = [ConfigurationSectionDownload]::new() -} - -class ConfigurationSectionCredentials { - [string] $Path -} - -class ConfigurationSectionServices { - [string] $FileApiBaseUrl - [string] $AuthenticationTokenApiBaseUrl -} - -class ConfigurationSectionDownload { - [string] $TenantId - [string] $Role - [string] $Path - [bool] $EnsureUniqueNames - [string] $Filter -} - -class FileInfo { - [string] $Id - [string] $Name - [long] $Size -} - -class FileNameInfo { - [string] $Name - [string] $Extension -} - -class Credentials { - [string] $ClientId - [string] $ClientSecret -} - -#endregion Models diff --git a/powershell/YouforceDeveloperPortal/download/README.md b/powershell/YouforceDeveloperPortal/download/README.md deleted file mode 100644 index e997fd2..0000000 --- a/powershell/YouforceDeveloperPortal/download/README.md +++ /dev/null @@ -1,163 +0,0 @@ -# File API PowerShell download examples - -A collection of examples to download files with the File API using **PowerShell**. - -## Prerequisites - -- The script was prepared for PowerShell version 5.1 or above. With lower versions it might not work properly. -- Files to be downloaded cannot be bigger than 2 GB. - -## Getting Started - -Download the **file-api-integration-examples** repository. - -Inside the **powershell\YouforceDeveloperPortal\download** folder you can find these files: -- **DownloadFiles.ps1**: Script example to download specified files. -- **config.xml**: Configuration of the **DownloadFiles.ps1** script. - -## Running Examples - -### Download files - -The script will download all the files matching the configuration. - -1. Open **config.xml** with any text editor (e.g. Notepad) and fill the parameters. -See **Understanding the configuration** section to understand the meaning of each parameter. -2. Run the **DownloadFiles.ps1** script with the desired parameters. - -The first time you execute the script, it will ask for your credentials and will save them securely in your computer in the path specified in the configuration. -The next executions will use the saved credentials unless you manually specify the opposite (see **Parameters** section). - -#### Parameters - -**`-ConfigPath`** -> Configuration full path. -> -> **Mandatory:** False -> **Default value:** {DownloadFiles.ps1 folder}\config.xml -> -> **Example:** -ConfigPath "C:\Users\Foorby\config.xml" - -**`-RenewCredentials`** -> Indicates if you want to renew the credentials saved in the system (true) or keep using the saved one (false). -> This parameter is useful in case you changed your client ID or client secret. -> In most of the cases you won't need this parameter set. -> -> **Mandatory:** False -> **Default value:** $false -> -> **Example:** -RenewCredentials $true - -#### Example 1. Download files using the default configuration path - -```powershell -& "C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\download\DownloadFiles.ps1" -``` - -#### Example 2. Download files specifying the configuration path - -```powershell -& "C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\download\DownloadFiles.ps1" -ConfigPath "C:\Users\Foorby\config.xml" -``` - -#### Example 3. Download files using new credentials - -```powershell -& "C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\download\DownloadFiles.ps1" -RenewCredentials $true -``` - -## Understanding the configuration - -Inside the **config.xml** file you will find these parameters: - -### Attributes of the `Credentials` element - -**`Path`** -> XML file path where the credentials will be storaged. -> :warning: It's important that the file you put in the path has an .xml extension, otherwise the example will not work properly. -> -> **Example:** C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\credentials\credentials_integration.xml - -### Attributes of the `Services` element - -**`FileApiBaseUrl`** -> File API base URL. -> -> It should be set to **https://fileapi.youforce.com/v1.0** - -
- -**`AuthenticationTokenApiBaseUrl`** -> Authentication token API base URL. -> -> It should be set to **https://api.raet.com/authentication** - -### Attributes of the `Download` element - -**`TenantId`** -> Tenant you will use to download the files. -> -> **Example:** 1122334 - -
- -**`Role`** -> Role of your application. -> -> Must be set to any of these values: -> **· Subscriber:** to download files provided to you (the most common scenario). -> **· Publisher:** to download files uploaded by you. - -
- -**`Path`** -> Path where the files will be downloaded. -> -> **Example:** C:\Visma\File API\Ftaas.Examples\powershell\download\output - -
- -**`EnsureUniqueNames`** -> Indicates if you want to rename the files to be unique before downloading them. -> -> Must be set to any of these values: -> **· false:** the downloaded file will replace any existing file with the same name. -> **· true:** the downloaded file will be renamed if there is any existing file with the same name. ->   Format: {original file name}_{timestamp}.{original extension} ->   Original: TestFile.txt ->   Renamed: TestFile_20220304T1229027372Z.txt - -
- -**`Filter`** -> If empty, all the available (not downloaded yet) files will be listed. -> You can learn more about filters in the [File API documentation](https://vr-api-integration.github.io/file-api-documentation/guides_search_for_files.html). -> -> **Example:** startsWith(FileName, 'employee_profile') and uploadDate gt 2022-02-08T11:02:00Z - -## Example of a valid configuration - -```xml - - - C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\credentials\credentials_integration1.xml - - - - https://fileapi.youforce.com/v1.0 - https://api.raet.com/authentication - - - - 1122334 - subscriber - C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\download\output - true - startsWith(FileName, 'employee_profile') and uploadDate gt 2022-02-08T11:02:00Z - - -``` - -## Authors - -**Visma - Transporters Team** diff --git a/powershell/YouforceDeveloperPortal/download/config.xml b/powershell/YouforceDeveloperPortal/download/config.xml deleted file mode 100644 index d6722d2..0000000 --- a/powershell/YouforceDeveloperPortal/download/config.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - EnterTheFilePathWhereYouWantToStoreYourCredentials (e.g. C:\Visma\File API\Ftaas.Examples\powershell\credentials\credentials_integration.xml) - - - - https://fileapi.youforce.com/v1.0 - https://api.raet.com/authentication - - - - EnterYourTenantId - subscriber - EnterTheFolderPathWhereYouWantToDownloadTheFilesTo (e.g. C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\download\output) - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/powershell/YouforceDeveloperPortal/upload/README.md b/powershell/YouforceDeveloperPortal/upload/README.md deleted file mode 100644 index 65ea740..0000000 --- a/powershell/YouforceDeveloperPortal/upload/README.md +++ /dev/null @@ -1,158 +0,0 @@ -# File API PowerShell upload examples - -A collection of examples to upload files with the File API using **PowerShell**. - -## Prerequisites - -- The script was prepared for PowerShell version 5.1 or above. With lower versions it might not work properly. -- Files to be uploaded cannot be bigger than 100 MB. - -## Getting Started - -Download the **file-api-integration-examples** repository. - -Inside the **powershell\YouforceDeveloperPortal\upload** folder you can find these files: - -- **UploadFile.ps1**: Script example to upload specified files. -- **config.xml**: Configuration of the **UploadFile.ps1** script. - -## Running Examples - -### Upload files - -The script will upload all the files matching the configuration. - -1. Open **config.xml** with any text editor (e.g. Notepad) and fill the parameters. -See **Understanding the configuration** section to understand the meaning of each parameter. -2. Run the **UploadFile.ps1** script with the desired parameters. - -The first time you execute the script, it will ask for your credentials and will save them securely in your computer in the path specified in the configuration. -The next executions will use the saved credentials unless you manually specify the opposite (see **Parameters** section). - -#### Parameters - -**`-ConfigPath`** -> Configuration full path. -> -> **Mandatory:** False -> **Default value:** {UploadFile.ps1 folder}\config.xml -> -> **Example:** -ConfigPath "C:\Users\Foorby\config.xml" - -**`-RenewCredentials`** -> Indicates if you want to renew the credentials saved in the system (true) or keep using the saved one (false). -> This parameter is useful in case you changed your client ID or client secret. -> In most of the cases you won't need this parameter set. -> -> **Mandatory:** False -> **Default value:** $false -> -> **Example:** -RenewCredentials $true - -#### Example 1. Upload files using the default configuration path - -```powershell -& "C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\upload\UploadFile.ps1" -``` - -#### Example 2. Upload files specifying the configuration path - -```powershell -& "C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\upload\UploadFile.ps1" -ConfigPath "C:\Users\Foorby\config.xml" -``` - -#### Example 3. Upload files using new credentials - -```powershell -& "C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\upload\UploadFile.ps1" -RenewCredentials $true -``` - -## Understanding the configuration - -Inside the **config.xml** file you will find these parameters: - -### Attributes of the `Credentials` element - -**`Path`** -> XML file path where the credentials will be stored. -> :warning: It's important that the file you put in the path has an .xml extension, otherwise the example will not work properly. -> -> **Example:** C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\credentials\credentials_integration.xml - -### Attributes of the `Services` element - -**`FileApiBaseUrl`** -> File API base URL. -> -> It should be set to **** - -
- -**`AuthenticationTokenApiBaseUrl`** -> Authentication token API base URL. -> -> It should be set to **** - -### Attributes of the `Upload` element - -**`TenantId`** -> Tenant you will use to upload the files. -> -> **Example:** 1122334 - -
- -**`BusinessTypeId`** -> The business type id of the file to upload. -> -> **Example:** 9890988 - -
- -**`Path`** -> Full path of the directory that contains the files to upload -> -> **Example:** C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\upload - -
- -**`Filter`** -> The filemask that will select the files to upload -> -> **Example:** data*.xml - -
- -**`ArchivePath`** -> Full path of the directory where sussessfully uploaded files will be archived to. -> -> **Example:** C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\archive - -
- -## Example of a valid configuration - -```xml - - - C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\credentials\credentials_integration1.xml - - - - https://fileapi.youforce.com/v1.0 - https://api.raet.com/authentication - - - - 1122334 - 9890988 - C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\upload - data*.xml - C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\archive - - -``` - -## Authors - -**Visma - Transporters Team** diff --git a/powershell/YouforceDeveloperPortal/upload/UploadFile.ps1 b/powershell/YouforceDeveloperPortal/upload/UploadFile.ps1 deleted file mode 100644 index 62cb292..0000000 --- a/powershell/YouforceDeveloperPortal/upload/UploadFile.ps1 +++ /dev/null @@ -1,604 +0,0 @@ -# This example shows how to upload a file. -# Authors: Visma - Transporters Team - -[CmdletBinding()] -Param( - [Alias("ConfigPath")] - [Parameter( - Mandatory = $false, - HelpMessage = 'Full filePath of the configuration (e.g. C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\upload\config.xml). Default value: set in the code.' - )] - [string] $_configPath, - - [Alias("RenewCredentials")] - [Parameter( - Mandatory = $false, - HelpMessage = 'Boolean. $true if you want to renew your credentials. $false otherwise' - )] - [bool] $_renewCredentials = $false -) - -$ErrorActionPreference = "Stop" - -# The default value of this parameter is set here because $PSScriptRoot is empty if used directly in Param() through PowerShell ISE. -if (-not $_configPath) { - $_configPath = "$($PSScriptRoot)\config.xml" -} - -Write-Host "=========================================================" -Write-Host "File API example: Upload a file." -Write-Host "=========================================================" - -Write-Host "(you can stop the script at any moment by pressing the buttons 'CTRL'+'C')" - -#region Configuration - -[ConfigurationManager] $configurationManager = [ConfigurationManager]::new() - -try { - $config = $configurationManager.Get($_configPath) -} -catch { - [Helper]::EndProgramWithError($_, "Failure retrieving the configuration. Tip: see the README.MD to check the format of the parameters.") -} - -#endregion Configuration - -#region Retrieve/Create credentials - -$credentialsManager = [CredentialsManager]::new($config.Credentials.Path) -$credentialsService = [CredentialsService]::new($credentialsManager) - -try { - if ($_renewCredentials) { - $credentials = $credentialsService.CreateNew() - } - else { - $credentials = $credentialsService.Retrieve() - } -} -catch { - [Helper]::EndProgramWithError($_, "Failure retrieving the credentials.") -} - -#endregion Retrieve/Create credentials - -#region Retrieve authentication token - -$authenticationApiClient = [AuthenticationApiClient]::new($config.Services.AuthenticationTokenApiBaseUrl) -$authenticationApiService = [AuthenticationApiService]::new($authenticationApiClient) - -try { - $token = $authenticationApiService.NewToken($credentials.ClientId, $credentials.ClientSecret) -} -catch { - [Helper]::EndProgramWithError($_, "Failure retrieving the authentication token.") -} - -#endregion Retrieve authentication token - -$fileApiClient = [FileApiClient]::new($config.Services.FileApiBaseUrl, $token) -$fileApiService = [FileApiService]::new($fileApiClient, $config.Upload.TenantId, $config.Upload.BusinessTypeId) - -#region Upload Directory contents - -Get-ChildItem -Path $config.Upload.Path -Filter $config.Upload.Filter | ForEach-Object -Process { - - $filenameToUpload = $_.FullName - - try { - $createdFilePath = $fileApiService.CreateFileToUpload($filenameToUpload) - $fileApiService.UploadFile($createdFilePath, $(Split-Path -Path $_.FullName -Leaf)) - } - catch { - [Helper]::EndProgramWithError($_, "Failure uploading file $($filenameToUpload).") - } - - try { - $archivedFile = [Helper]::ArchiveFile($config.Upload.ArchivePath, $_.FullName) - } - catch { - [Helper]::EndProgramWithError($_, "Failure archiving file to $($archivedFile).") - } - -} - -#endregion Upload Directory contents - -[Helper]::EndProgram() - -# -------- END OF THE PROGRAM -------- -# Below there are classes and models to help the readability of the program - -#region Helper classes - -class ConfigurationManager { - [Configuration] Get($configPath) { - Write-Host "----" - Write-Host "Retrieving the configuration." - - if (-not (Test-Path $configPath -PathType Leaf)) { - throw "Configuration not found.`r`n| Path: $($configPath)" - } - - $configDocument = [xml](Get-Content $configPath) - $config = $configDocument.Configuration - - $credentialsPath = $config.Credentials.Path - - $fileApiBaseUrl = $config.Services.FileApiBaseUrl - $authenticationTokenApiBaseUrl = $config.Services.AuthenticationTokenApiBaseUrl - - $tenantId = $config.Upload.TenantId - $businessTypeId = $config.Upload.BusinessTypeId - $contentDirectoryPath = $config.Upload.Path - $contentFilter = $config.Upload.Filter - - $archivePath = $config.Upload.ArchivePath - - $missingConfiguration = @() - if ([string]::IsNullOrEmpty($credentialsPath)) { $missingConfiguration += "Credentials.Path" } - if ([string]::IsNullOrEmpty($fileApiBaseUrl)) { $missingConfiguration += "Services.FileApiBaseUrl" } - if ([string]::IsNullOrEmpty($authenticationTokenApiBaseUrl)) { $missingConfiguration += "Services.AuthenticationTokenApiBaseUrl" } - if ([string]::IsNullOrEmpty($tenantId)) { $missingConfiguration += "Upload.TenantId" } - if ([string]::IsNullOrEmpty($businessTypeId)) { $missingConfiguration += "Upload.BusinessTypeId" } - if ([string]::IsNullOrEmpty($contentDirectoryPath)) { $missingConfiguration += "Upload.Path" } - if ([string]::IsNullOrEmpty($contentFilter)) { $missingConfiguration += "Upload.Filter" } - if ([string]::IsNullOrEmpty($archivePath)) { $missingConfiguration += "Upload.ArchivePath" } - - if ($missingConfiguration.Count -gt 0) { - throw "Missing parameters: $($missingConfiguration -Join ", ")" - } - - $wrongConfiguration = @() - if (-not [Validator]::IsPath($credentialsPath)) { $wrongConfiguration += "Credentials.Path" } - if (-not [Validator]::IsUri($fileApiBaseUrl)) { $wrongConfiguration += "Services.FileApiBaseUrl" } - if (-not [Validator]::IsUri($authenticationTokenApiBaseUrl)) { $wrongConfiguration += "Services.AuthenticationTokenApiBaseUrl" } - if (-not [Validator]::IsPath($contentDirectoryPath)) { $wrongConfiguration += "Upload.Path" } - if (-not [Validator]::IsPath($archivePath)) { $wrongConfiguration += "Upload.ArchivePath" } - - if ($wrongConfiguration.Count -gt 0) { - throw "Wrong configured parameters: $($wrongConfiguration -Join ", ")" - } - - $configuration = [Configuration]::new() - $configuration.Credentials.Path = $credentialsPath - $configuration.Services.FileApiBaseUrl = $fileApiBaseUrl - $configuration.Services.AuthenticationTokenApiBaseUrl = $authenticationTokenApiBaseUrl - $configuration.Upload.TenantId = $tenantId - $configuration.Upload.BusinessTypeId = $businessTypeId - $configuration.Upload.Path = $contentDirectoryPath - $configuration.Upload.Filter = $contentFilter - $configuration.Upload.ArchivePath = $archivePath - - Write-Host "Configuration retrieved." - - return $configuration - } -} - -class CredentialsService { - hidden [CredentialsManager] $_credentialsManager - - CredentialsService ([CredentialsManager] $manager) { - $this._credentialsManager = $manager - } - - [Credentials] Retrieve() { - $credentials = $this._credentialsManager.Retrieve() - if ($null -eq $credentials) { - $credentials = $this.CreateNew() - return $credentials - } - - return $credentials - } - - [Credentials] CreateNew() { - $this._credentialsManager.CreateNew() - $credentials = $this._credentialsManager.Retrieve() - - return $credentials - } -} - -class CredentialsManager { - hidden [string] $_credentialsPath - - CredentialsManager([string] $storagePath) { - $this._credentialsPath = $storagePath - } - - [void] CreateNew() { - $storagePath = Split-Path $this._credentialsPath - - Write-Host "----" - Write-Host "Saving your credentials." - Write-Host "| Path: $($this._credentialsPath)" - - if (-not (Test-Path -Path $storagePath -PathType Container)) { - Write-Host "----" - Write-Host "Storage credential filePath doesn't exist. Creating it." - Write-Host "| Path: $($storagePath)" - - New-Item -ItemType Directory -Force -Path $storagePath - } - - Write-Host "Enter your credentials." - $clientId = Read-Host -Prompt '| Client ID' - $clientSecret = Read-Host -Prompt '| Client secret' -AsSecureString - - [PSCredential]::new($clientId, $clientSecret) | Export-CliXml -Path $this._credentialsPath - - Write-Host "----" - Write-Host "Credentials saved." - } - - [Credentials] Retrieve() { - Write-Host "----" - Write-Host "Retrieving your credentials." - Write-Host "| Path: $($this._credentialsPath)" - - if (-not (Test-Path -Path $this._credentialsPath -PathType Leaf)) { - Write-Host "----" - Write-Host "Credentials not found." - Write-Host "| Path: $($this._credentialsPath)" - - return $null - } - - $credentialsStorage = Import-CliXml -Path $this._credentialsPath - - $credentials = [Credentials]::new() - $credentials.ClientId = $credentialsStorage.GetNetworkCredential().UserName - $credentials.ClientSecret = $credentialsStorage.GetNetworkCredential().Password - - Write-Host "Credentials retrieved." - - return $credentials - } -} - -class FileApiService { - hidden [FileApiClient] $_fileApiClient - hidden [string] $_tenantId - hidden [long] $_uploadSizeLimit - hidden [string] $_boundary - hidden [string] $_businessTypeId - - FileApiService( - [FileApiClient] $fileApiClient, - [string] $tenantId, - [string] $businessTypeId - ) { - $this._fileApiClient = $fileApiClient - $this._tenantId = $tenantId - $this._boundary = "file_info" - $this._businessTypeId = $businessTypeId - - # API supports files up to 100 megabytes - $this._uploadSizeLimit = 100 * 1024 * 1024 - } - - [string] CreateFileToUpload([string] $filename) { - Write-Host "----" - Write-Host "Creating a bundle with the file $($filename) to upload." - $headerFilePath = "" - $footerFilePath = "" - try { - $folderPath = $(Split-Path -Path $filename) - $contentFilename = $(Split-Path -Path $filename -Leaf) - $createdFilePath = "$($folderPath)\$([Helper]::ConvertToUniqueFilename("multipart.bin"))" - - $headerFilename = [Helper]::ConvertToUniqueFilename("header.txt") - $headerFilePath = "$($folderPath)\$($headerFilename)" - $headerContent = "--$($this._boundary)`r`n" # Windows line breaks are required. - $headerContent += "Content-Type: application/json; charset=UTF-8`r`n" - $headerContent += "`r`n" - $headerContent += "{`r`n`"name`":`"$($contentFilename)`",`r`n`"businesstypeid`":`"$($this._businessTypeId)`"`r`n}`r`n" - $headerContent += "--$($this._boundary)`r`n`r`n" - - $footerFilename = [Helper]::ConvertToUniqueFilename("footer.txt") - $footerFilePath = "$($folderPath)\$($footerFilename)" - $footerContent = "`r`n--$($this._boundary)--" - - New-Item -Path $folderPath -Name $headerFilename -Value $headerContent - New-Item -Path $folderPath -Name $footerFilename -Value $footerContent - - cmd /c copy /b $headerFilePath + $filename + $footerFilePath $createdFilePath - Write-Host "File created." - return $createdFilePath - } - finally { - if (Test-Path $headerFilePath) { - Remove-Item -Force -Path $headerFilePath - } - if (Test-Path $footerFilePath) { - Remove-Item -Force -Path $footerFilePath - } - } - } - - [void] UploadFile([string] $filePath, $originalFilename) { - if ((Get-Item $filePath).Length -gt $this._uploadSizeLimit) { - throw "Cannot upload files bigger $($this._uploadSizeLimit) bytes." - } - Write-Host "----" - Write-Host "Uploading the file." - Write-Host "| File: $($originalFilename)" - Write-Host "| Business type: $($this._businessTypeId)" - $this._fileApiClient.UploadFile($this._tenantId, $filePath, $this._boundary) - - Write-Host "File uploaded." - } -} - -class FileApiClient { - [string] $BaseUrl - - hidden [PSCustomObject] $_defaultHeaders - - FileApiClient ( - [string] $baseUrl, - [string] $token - ) { - $this.BaseUrl = $baseUrl - $this._defaultHeaders = @{ - "Authorization" = "Bearer $($token)"; - } - } - - [PSCustomObject] UploadFile([string] $tenantId, [string] $multipartcontentFilePath, [string] $boundary) { - $headers = $this._defaultHeaders - $headers["x-raet-tenant-id"] = $tenantId - $headers["Content-Type"] = "multipart/related;boundary=$($boundary)" - try { - $response = Invoke-RestMethod ` - -Method "Post" ` - -Uri "$($this.BaseUrl)/files?uploadType=multipart" ` - -Headers $headers ` - -InFile "$($multipartcontentFilePath)" - - return $response - } - - finally { - if (Test-Path $multipartcontentFilePath) { - Remove-Item -Force -Path $multipartcontentFilePath - } - } - } -} - -class AuthenticationApiService { - hidden [AuthenticationApiClient] $_authenticationApiClient - - AuthenticationApiService([AuthenticationApiClient] $authenticationApiClient) { - $this._authenticationApiClient = $authenticationApiClient - } - - [string] NewToken([string] $clientId, [string] $clientSecret) { - Write-Host "----" - Write-Host "Retrieving the authentication token." - - $response = $this._authenticationApiClient.NewToken($clientId, $clientSecret) - $token = $response.access_token - - Write-Host "Authentication token retrieved." - - return $token - } -} - -class AuthenticationApiClient { - hidden [string] $_baseUrl - - AuthenticationApiClient([string] $baseUrl) { - $this._baseUrl = $baseUrl - } - - [PSCustomObject] NewToken([string] $clientId, [string] $clientSecret) { - $headers = @{ - "Content-Type" = "application/x-www-form-urlencoded"; - "Cache-Control" = "no-cache"; - } - $body = @{ - "grant_type" = "client_credentials"; - "client_id" = $clientId; - "client_secret" = $clientSecret; - } - - $response = Invoke-RestMethod ` - -Method "Post" ` - -Uri "$($this._baseUrl)/token" ` - -Headers $headers ` - -Body $body - - return $response - } -} - -class Validator { - static [bool] IsUri([string] $testParameter) { - try { - $uri = $testParameter -As [System.Uri] - - if ($uri.AbsoluteUri) { - return $true - } - else { - return $false - } - } - catch { - return $false - } - } - - static [bool] IsBool([string] $testParameter) { - try { - $result = [bool]::TryParse($testParameter, [ref] $null) - return $result - } - catch { - return $false - } - } - - static [bool] IsPath([string] $testParameter) { - try { - $result = Test-Path $testParameter -IsValid - return $result - } - catch { - return $false - } - } -} - -class Helper { - static [string] ArchiveFile([string] $archivePath, [string] $filename) { - $filenameInfo = [Helper]::GetFilenameInfo($filename) - $uniqueArchiveFilename = [Helper]::ConvertToUniqueFilename($filenameInfo.FullName) - $archivePath = Join-Path $($archivePath) $($uniqueArchiveFilename) - try { - Move-Item $filename -Destination $archivePath - Write-Host "File archived to ($($archivePath))." - } - catch { - Write-Host "File NOT archived." - throw $_ - } - return $archivePath - } - - static [string] ConvertToUniqueFilename([string] $filename) { - $filenameInfo = [Helper]::GetFilenameInfo($filename) - $filePath = $filenameInfo.Path - $filenameWithoutExtension = $filenameInfo.Name - $fileExtension = $filenameInfo.Extension - $timestamp = Get-Date -Format "yyyyMMdd_HHmmss_fff" - if([string]::IsNullOrEmpty($filepath)){ - $uniqueFilename = "$($filenameWithoutExtension)_$($timestamp)$($fileExtension)" - } - else { - $uniqueFilename = "$($filePath)\\$($filenameWithoutExtension)_$($timestamp)$($fileExtension)" - } - return $uniqueFilename - } - - static [FilenameInfo] GetFilenameInfo([string] $filename) { - $filenameInfo = [FilenameInfo]::new() - $filenameInfo.FullPath = $filename - $filenameInfo.FullName = "" - $filenameInfo.Name = "" - $filenameInfo.Extension = "" - $filenameInfo.Path = "" - - $splitPath = $filename -split "\\" - if ($splitPath.Length -gt 1) { - $filenameInfo.Path = $splitPath[0..($splitPath.Length - 2)] -Join "\" - $filenameInfo.FullName = $splitPath[$splitPath.Length - 1] - $filenameInfo.Name = $filenameInfo.FullName - } - else { - $filenameInfo.Name = $filename - $filenameInfo.FullName = $filename - } - - $splitFilename = $filenameInfo.FullName -split "\." - if($splitFilename.Length -gt 1) { - $filenameInfo.Name = $splitFilename[0] - $filenameInfo.Extension = ".$($splitFilename[-1])" - } - - return $filenameInfo - } - - static [void] EndProgram() { - [Helper]::FinishProgram($false) - } - - static [void] EndProgramWithError([System.Management.Automation.ErrorRecord] $errorRecord, [string] $genericErrorMessage) { - Write-Host "ERROR - $($genericErrorMessage)" -ForegroundColor "Red" - - $errorMessage = "Unknown error." - if ($errorRecord.ErrorDetails.Message) { - $errorDetails = $errorRecord.ErrorDetails.Message | ConvertFrom-Json - if ($errorDetails.message) { - $errorMessage = $errorDetails.message - } - elseif ($errorDetails.error.message) { - $errorMessage = $errorDetails.error.message - } - } - elseif ($errorRecord.Exception.message) { - $errorMessage = $errorRecord.Exception.message - } - - Write-Host "| Error message: $($errorMessage)" -ForegroundColor "Red" - Write-Host "| Error line in the script: $($errorRecord.InvocationInfo.ScriptLineNumber)" -ForegroundColor "Red" - - [Helper]::FinishProgram($true) - } - - hidden static [void] FinishProgram([bool] $finishWithError) { - Write-Host "----" - Write-Host "End of the example." - - if ($finishWithError) { - exit 1 - } - else { - exit - } - } -} - -#endregion Helper classes - -#region Models - -class Configuration { - $Credentials = [ConfigurationSectionCredentials]::new() - $Services = [ConfigurationSectionServices]::new() - $Upload = [ConfigurationSectionUpload]::new() -} - -class ConfigurationSectionCredentials { - [string] $Path -} - -class ConfigurationSectionServices { - [string] $FileApiBaseUrl - [string] $AuthenticationTokenApiBaseUrl -} - -class ConfigurationSectionUpload { - [string] $TenantId - [string] $BusinessTypeId - [string] $Path - [string] $Filter - [string] $ArchivePath -} - -class FileInfo { - [string] $Id - [string] $Name - [long] $Size -} - -class FilenameInfo { - [string] $Name - [string] $Extension - [string] $FullName - [string] $Path - [string] $FullPath -} - -class Credentials { - [string] $ClientId - [string] $ClientSecret -} - -#endregion Models diff --git a/powershell/YouforceDeveloperPortal/upload/config.xml b/powershell/YouforceDeveloperPortal/upload/config.xml deleted file mode 100644 index f45885f..0000000 --- a/powershell/YouforceDeveloperPortal/upload/config.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - EnterTheFilePathWhereYouWantToStoreYourCredentials (e.g. C:\Visma\File API\Ftaas.Examples\powershell\credentials\credentials_integration.xml) - - - - https://fileapi.youforce.com/v1.0 - https://api.raet.com/authentication - - - - EnterYourTenantId - EnterYourBusinessTypeId - EnterTheDirectoryToUpload (e.g.: C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\upload\file) - EnterFilemaskToUseForUpload (e.g.: *.xml) - EnterTheDirectoryForArchivedFiles (e.g.: C:\Visma\File API\Ftaas.Examples\powershell\YouforceDeveloperPortal\archive) - - \ No newline at end of file