Skip to content
Permalink
Branch: master
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
123 lines (105 sloc) 5.04 KB
Workflow BlobSync {
$PSDisableSerializationPreference = $true
$MaxReturn = 400; #experiment with this value -- this will drive the number of files pulled at at time from the source
[int64]$Total = 0;
$Token = $null;
#init to int64 as I was working with large containers of files
[int64]$FilesTransferred = 0;
[int64]$FilesTransferSuccess = 0;
[int64]$FilesTransferFail = 0;
$runflag = $TRUE;
#Populate the values below as necessary -- script assumes the same container will be used in the source and destination
[string]$containerName = 'SourceAndDestinationContainer'
$Srcstorageaccountkey = "SrcKey";
$srcStorageAccount = "SrcStorageAccount";
$deststorageAccount = "DestStorageAccount";
$DestAccountKey = "Destination AccountKey";
$TimeStart = Get-Date -format HH:mm:ss
"Destination Array Population...";
$DestBlobsHash = @{};
$DestBlobsHash = InlineScript
{
$DestBlobsHashTemp = @{}
$destContext = New-AzureStorageContext -StorageAccountName $using:deststorageAccount -StorageAccountKey $using:DestAccountKey
Get-AzureStorageBlob -Context $destContext -Container $using:containerName -ContinuationToken $Token |
Select-Object -Property Name, LastModified, ContinuationToken |
ForEach { $DestBlobsHashTemp[$_.Name] = $_.LastModified.UtcDateTime };
$DestBlobsHashTemp ;
}
DO {
"Saving State...";
Checkpoint-Workflow #on each itteration persist our state
"Source Array Population...";
$SrcBlobs = InlineScript
{
$sourceContext = New-AzureStorageContext -StorageAccountName $Using:srcStorageAccount -StorageAccountKey $using:Srcstorageaccountkey
$SrcBlobs = Get-AzureStorageBlob -Context $sourceContext -Container $Using:containerName -MaxCount $using:MaxReturn -ContinuationToken $using:Token |
Select-Object -Property Name, LastModified, ContinuationToken;
if ($SrcBlobs -ne $null) {
$cnt = $SrcBlobs.Count;
write-host " ** Files Found: $cnt - Pulling $using:MaxReturn files at a time...";
}
$SrcBlobs;
}
$Total += $SrcBlobs.Count
if ($SrcBlobs.Length -le 0) {
$runflag = $false;
}
$Token = $SrcBlobs[$SrcBlobs.Count - 1].ContinuationToken;
#experiment with the throttle -- I found 25 to work well within Azure automation for invoking copy operations
ForEach -parallel -ThrottleLimit 25 ($SrcBlob in $SrcBlobs) {
# search in destination blobs for the source blob and unmodified, if found copy it
$CopyThisBlob = $false
if ($SrcBlob.Name -eq $null) {
"SrcBlob name is Null?";
}
elseif ($DestBlobsHash.count -eq 0) {
$CopyThisBlob = $true
}
elseif (!($DestBlobsHash.Contains($SrcBlob.Name))) {
$CopyThisBlob = $true
}
elseif ($SrcBlob.LastModified.UtcDateTime -gt $DestBlobsHash.Item($SrcBlob.Name)) {
$CopyThisBlob = $true
}
if ($CopyThisBlob) {
#Start copying the blobs to destination container
[string]$blobToCopy = $SrcBlob.Name
$UtcTime = Get-Date;
" ++ Copying: $blobToCopy - " + $UtcTime.ToUniversalTime();
$workflow:FilesTransferred++;
try {
inlinescript {
$destContext = New-AzureStorageContext -StorageAccountName $using:deststorageAccount -StorageAccountKey $using:DestAccountKey
$sourceContext = New-AzureStorageContext -StorageAccountName $using:srcStorageAccount -StorageAccountKey $using:Srcstorageaccountkey
$c = Start-AzureStorageBlobCopy -SrcContainer $using:containerName -SrcBlob $using:blobToCopy -DestContainer $using:containerName -DestBlob $using:blobToCopy -SrcContext $sourceContext -DestContext $destContext -Force
}
$workflow:FilesTransferSuccess++;
}
catch {
Write-Error "$using:blobToCopy transfer failed"
$workflow:FilesTransferFail++;
}
}
}
}
While (($Token -ne $Null) - ($runflag -eq $TRUE))
$TimeEnd = Get-Date -format HH:mm:ss
$TimeDiff = New-TimeSpan $TimeStart $TimeEnd
if ($TimeDiff.Seconds -lt 0) {
$Hrs = ($TimeDiff.Hours) + 23
$Mins = ($TimeDiff.Minutes) + 59
$Secs = ($TimeDiff.Seconds) + 59
}
else {
$Hrs = $TimeDiff.Hours
$Mins = $TimeDiff.Minutes
$Secs = $TimeDiff.Seconds
}
$Difference = '{0:00}:{1:00}:{2:00}' -f $Hrs, $Mins, $Secs
"Total blobs in container $container : $Total"
"Total files transferred: $FilesTransferred"
"Transfered successfully: $FilesTransferSuccess"
"Transfer failed: $FilesTransferFail"
"Elapsed time: $Difference `n"
}
You can’t perform that action at this time.