get-ftp.ps1

function Get-FTP
{
    <#
    .Synopsis
        Gets files from FTP
    .Description
        Lists files on an FTP server, or downloads files
    .Example
        Get-FTP -FTP "ftp://edgar.sec.gov/edgar/full-index/1999/" -Download -Filter "*.idx", "*.xml"
    .Example
        Get-FTP -FTP "ftp://edgar.sec.gov/edgar/full-index/1999/" -Download -Filter "*.idx", "*.xml"  -DownloadAsJob
    .Link
        Push-FTP
    #>
    [OutputType([IO.FileInfo])]
    [CmdletBinding(DefaultParameterSetName='FTPSite')]
    param(
    # The root url of an FTP server
    [Parameter(Mandatory=$true,Position=0,ValueFromPipelineByPropertyName=$true,ParameterSetName='FTPSite')]
    [Alias('FTP')]
    [Uri]$FtpRoot,

    # A list of specific files on an FTP server.  Useful for when dealing with FTP servers that do not allow listing.
    [Parameter(Mandatory=$true,Position=0,ValueFromPipeline=$true,ValueFromPipelineByPropertyName=$true,ParameterSetName='FTPFile')]
    [Uri[]]
    $FtpFile,

    # The credential used to connect to FTP.  If not provided, will connect anonymously.
    [Parameter(ValueFromPipelineByPropertyName=$true)]
    [Management.Automation.PSCredential]
    $Credential,

    # If set, will download files instead of discover them
    [Parameter(ValueFromPipelineByPropertyName=$true,ParameterSetName='FTPSite')]
    [Switch]$Download,

    # The download path (by default, the downloads directory)
    [Parameter(ValueFromPipelineByPropertyName=$true)]
    [string]$DownloadPath = "$env:UserProfile\Downloads",

    # If provided, will only download files that match the filter
    [Parameter(ValueFromPipelineByPropertyName=$true,Position=1,ParameterSetName='FTPSite')]
    [string[]]$Filter,

    # If set, will download files that already have been downloaded and have the exact same file size.
    [Parameter(ValueFromPipelineByPropertyName=$true,ParameterSetName='FTPSite')]
    [Switch]$Force,

    # If set, downloads will run as a background job
    [Parameter(ValueFromPipelineByPropertyName=$true)]
    [Switch]
    $DownloadAsJob,

    # If set, downloads will be run in parallel in a PowerShell workflow
    [Parameter(ValueFromPipelineByPropertyName=$true)]
    [Switch]
    $UseWorkflow,

    # If set, download progress will not be displayed
    [Parameter(ValueFromPipelineByPropertyName=$true)]
    [Switch]
    $HideProgress,

    # The size of the copy buffer.  By default, this is 50kb
    [Uint32]
    $BufferSize = 50kb


    )

    begin {
       
        $folders = New-Object "system.collections.generic.queue[string]"
        $files= New-Object "system.collections.generic.queue[string]"
       
        $GetFtpStream = {
            param($url, $method, $Credential)
            try {
               
                $ftpRequest = [System.Net.FtpWebRequest]::Create($url)
                if ($Credential)  {
                    $ftpRequest.Credentials = $Credential.GetNetworkCredential()
                }
                $ftpRequest.Method = $method
                $ftpresponse = $ftpRequest.GetResponse()
                $reader = New-Object IO.StreamReader $ftpresponse.GetResponseStream()

                while (($line = $reader.ReadLine()) -ne $null) {
                    $line.Trim()
                }
                if ($reader) {
                    $reader.Dispose()
                }
               
                if ($ftpresponse.Close) {
                    $ftpresponse.Close()
                }
            } catch {
                $err = $_
                Write-Error -Exception $err.Exception -Message "Error in FTP $method Request on $url"
                return
            } finally {
            }
           
        }

        $GetFtpFile = {
           
            param($Source,$Target,$Credential, $HideProgress, $BufferSize)
            try {
           
            $FileSize =
                try {
                    $ftprequest = [Net.FtpWebRequest]::create($Source)
                    if ($Credential) {
                        $ftprequest.Credentials = $Credential.GetNetworkCredential()
                    }
                    $ftprequest.Method = [Net.WebRequestMethods+Ftp]::GetFileSize
                    $ftpresponse = $ftprequest.GetResponse()
                    $ftpresponse.ContentLength
                    if ($ftpresponse.Close) {
                        $ftpresponse.Close()
                    }
                } catch {

                }

           
 
            $ftprequest = [Net.FtpWebRequest]::create($Source)
            if ($Credential) {
                $ftprequest.Credentials = $Credential.GetNetworkCredential()
            }
            $ftprequest.Method = [Net.WebRequestMethods+Ftp]::DownloadFile
            $ftprequest.UseBinary = $true
            $ftprequest.KeepAlive = $false
      

            $ftpresponse = $ftprequest.GetResponse()

            $responsestream = $ftpresponse.GetResponseStream()
            if (-not $responsestream) { return  }
      
            $targetfile = New-Object IO.FileStream ($Target,[IO.FileMode]::Create)
            [byte[]]$readbuffer = New-Object byte[] $BufferSize
      
 
            $perc = 0
            $progressId = Get-Random
            do{
                $readlength = $responsestream.Read($readbuffer,0,$BufferSize)

               
                $targetfile.Write($readbuffer,0,$readlength)
                if ($FileSize) {
                    if (-not $HideProgress) {
                        $perc = $targetfile.Position * 100 / $FileSize
                        Write-Progress "Downloading $Source" "To $Target" -PercentComplete $perc -Id $progressId
                    }
                } else {
                    if (-not $HideProgress) {
                        $perc += 5
                        if ($perc -gt 100) { $perc = 0 }
                        Write-Progress "Downloading $Source" "To $Target" -PercentComplete $perc -Id $progressId
                    }
                }
            } while ($readlength -ne 0)

     
            $targetfile.close()

            if ($ftpresponse.Close) {
                $ftpresponse.Close()
            }
            Write-Progress "Downloading $Source" "To $Target" -Completed -Id $progressId

            Get-Item -Path $target

            } catch {
                $err = $_
                Write-Error -Exception $err.Exception -Message "FTP Error Downloading $source - $($err.Exception.Message)"
                return
            }           
        }                       
 
        $jobDefintion = [ScriptBlock]::Create(@"
param([Hashtable]`$Parameter)
`$getFtpFile = { $GetFtpFile }

& `$getFtpFile @parameter
"@)


        $workflowDefinition = @"
workflow getFtpFilesWorkflow(
    [Parameter(Position=0)]
    [Hashtable[]]`$ftpFileInput
    ) {
    foreach -parallel (`$ftpFile in `$ftpFileInput) {
        `$ftpFile | 
            inlineScript {
                `$parameter = `$(`$input)
                & { $GetFtpFile } @parameter 
            
            }
    }
}
"@

        . ([ScriptBlock]::create($workflowDefinition))

        $Ftpjobs = @()
       $AsyncDownloadPaths = @()
    }
    process {
       
        if ($PSCmdlet.ParameterSetName -eq 'FTPSite') {
            $null = $folders.Enqueue("$ftpRoot")
        } elseif ($PSCmdlet.ParameterSetName -eq 'FTPFile') {
            foreach ($f in $FtpFile) {
                $null = $files.Enqueue("$f")
            }
        }
       
        
    }

    end {
        $workFlowInputData = @()
        while($folders.Count -gt 0 -or
              $RunningFtpjobs.Count -gt 0 -or
              $files.Count -gt 0){

            if ($PSCmdlet.ParameterSetName -eq 'FTPSite' -and $folders.Count) {
               
                $fld = $folders.Dequeue()
       
                $newFiles = New-Object "system.collections.generic.list[string]"
                $newDirs = New-Object "system.collections.generic.list[string]"
                $operation = [System.Net.WebRequestMethods+Ftp]::ListDirectory
       
                foreach ($line in . $GetFtpStream $fld $operation $Credential 2>&1) {
                    if ($line -is [Management.Automation.ErrorRecord]) {
                        $line | Write-Error
                    } else {
                        [void]$newFiles.Add($line.Trim())
                    }
                }
                                                                   
                $operation = [System.Net.WebRequestMethods+Ftp]::ListDirectoryDetails
                foreach ($line in . $GetFtpStream $fld $operation $Credential 2>&1) {
                    if ($line -is [Management.Automation.ErrorRecord]) {
                        $line | Write-Error
                    } else {
                        [void]$newDirs.Add($line.Trim())
                    }
                   
                }
                   
                
                foreach ($d in $newDirs) {
                    $parts = @($d -split " " -ne '')




                    if ($parts.Count -eq 2) {
                        # First line, purely informational
                        continue
                    }


                    if ($parts.Count -eq 9) {
                        # 9 parts.  It's likely that this is a linux based FTP
                        # The last part should be the file name
                        # The preceeding 3 parts should be the modification time
                        # The preceeding 1 part should be the file size
                        if ($parts[-1] -eq '.' -or $parts[-1] -eq '..') {
                            continue
                        }
                        
                        $FileName = $parts[-1]
                        $FileSize = $parts[-5]
                        $FileDate = ((@($parts[-4..-3]) + (Get-Date).Year +  $parts[-2]) -join ' ') -as [datetime]
                                                


                        
                    } elseif ($parts.Count -eq 4) {
                        # First two parts should be date
                        # Third part should be file length
                        # Last part should be file name

                        $FileName= $parts[-1]
                        $FileSize = $parts[-2]
                        $FileDate = ($parts[0..1] -join ' ') -as [DateTime]
                    }


                    if (-not $FileName) {continue } 

                    if ($FileSize -eq 4096 -or -not $FileSize) {
                        $newName = $parts[-1]
                        Write-Verbose "Enqueing Folder $($fld + $FileName + "/")"
                        $null = $folders.Enqueue($fld + $FileName  + "/")
                    }


                    $out =
                        New-Object PSObject -Property @{
                            Ftp = $fld + "/" + $FileName
                            Size = $FileSize
                            UpdatedAt = $FileDate
                        }

                    if ($filter) {
                        $matched = $false
                        foreach ($f in $filter) {
                            if ($FileName -like "$f") {
                                $matched  = $true
                                break
                            }
                        }
                        if (-not $matched) {
                            continue
                        }
                    }
                    if ($download -or $psBoundParameters.DownloadPath) {
                       
                            $folderUri = [uri]("$fld".TrimEnd("/") + "/" + $FileName)
                       
                            $downloadTo = Join-Path $DownloadPath $folderUri.LocalPath
                            $downloadDir  = Split-Path $downloadTo
                            if (-not (Test-Path $downloadDir)) {
                                $null = New-Item -ItemType Directory $downloadDir
                            }

                            $item = Get-Item -Path $downloadTo -ErrorAction SilentlyContinue
                            if (($item.Length -ne $FileSize) -or $Force) {
                                if ($DownloadAsJob) {
                                   
                                
                                    $ftpJobs += Start-Job -ArgumentList @{
                                        Source =$folderUri
                                        Target = $downloadTo
                                        Credential = $Credential
                                        HideProgress = $HideProgress
                                        BufferSize = $BufferSize
                                    } -ScriptBlock $jobDefintion
                                } elseif ($UseWorkflow) {
                                    $workFlowInputData += @{
                                        Source =$folderUri
                                        Target = $downloadTo
                                        Credential = $Credential
                                        HideProgress = $HideProgress
                                        BufferSize = $BufferSize

                                    }
                                
                                } else {
                                    & $GetFtpFile -Source $folderUri -Target $downloadTo -Credential:$Credential -BufferSize $BufferSize -HideProgress:$HideProgress
                                }
                           
                            }
                           
                        
                        } else {

                        $out
                    }

                   
                                       
                                   
                }

               
            } elseif ($PSCmdlet.ParameterSetName  -eq 'FTPFile' -and $files.Count) {
                $file= $files.Dequeue()
                $folderUri  =[URI]$file
                $downloadTo = Join-Path $DownloadPath $folderUri.LocalPath
                $downloadDir  = Split-Path $downloadTo
                if (-not (Test-Path $downloadDir)) {
                    $null = New-Item -ItemType Directory $downloadDir
                }

 
                if ($DownloadAsJob) {                                                   
                    $ftpJobs += Start-Job -ArgumentList @{
                        Source =$folderUri
                        Target = $downloadTo
                        Credential = $Credential
                        HideProgress = $HideProgress
                        BufferSize = $BufferSize

                    } -ScriptBlock $jobDefintion
                } elseif ($UseWorkflow) {
                    $workFlowInputData += @{
                        Source =$folderUri
                        Target = $downloadTo
                        Credential = $Credential
                        HideProgress = $HideProgress
                        BufferSize = $BufferSize

                    }
                } else {
                   
                    $FileResults = & $GetFtpFile -Source $folderUri -Target $downloadTo -Credential:$Credential -BufferSize $BufferSize -HideProgress:$HideProgress 2>&1
                    if ($FileResults -is [Management.Automation.ErrorRecord]) {
                        $FileResults | Write-Error
                    } else {
                        $FileResults
                    }
                   
                }
             
            }
            
            if ($Ftpjobs) {
                $Ftpjobs | Receive-Job                               
                $RunningFtpjobs = @($Ftpjobs | Where-Object { $_.JobStateInfo.State -ne 'Completed' })               
            }
        }

        while ($workFlowInputData.Count -or $RunningFtpjobs) {
            if ($workFlowInputData) {
                $Ftpjobs += getFtpFilesWorkflow $workFlowInputData -asjob
            }
            $workFlowInputData = $null
            if ($Ftpjobs) {
                $Ftpjobs | Receive-Job                               
                $RunningFtpjobs = @($Ftpjobs | Where-Object { $_.JobStateInfo.State -ne 'Completed' })               
            }
        }

        if ($Ftpjobs) {
            $Ftpjobs | Receive-Job
            $Ftpjobs | Remove-Job
        }

 
    }
}