diff --git a/src/Accounts/Authentication/Utilities/CustomAssemblyResolver.cs b/src/Accounts/Authentication/Utilities/CustomAssemblyResolver.cs index ac933cc98c76..8e2c69c1bfdc 100644 --- a/src/Accounts/Authentication/Utilities/CustomAssemblyResolver.cs +++ b/src/Accounts/Authentication/Utilities/CustomAssemblyResolver.cs @@ -10,7 +10,7 @@ public static class CustomAssemblyResolver private static IDictionary NetFxPreloadAssemblies = new Dictionary(StringComparer.InvariantCultureIgnoreCase) { - {"Azure.Core", new Version("1.2.2.0")}, + {"Azure.Core", new Version("1.3.0.0")}, {"Microsoft.Bcl.AsyncInterfaces", new Version("1.0.0.0")}, {"Microsoft.IdentityModel.Clients.ActiveDirectory", new Version("3.19.2.6005")}, {"Microsoft.IdentityModel.Clients.ActiveDirectory.Platform", new Version("3.19.2.6005")}, diff --git a/src/Storage/Storage.Management.Test/Storage.Management.Test.csproj b/src/Storage/Storage.Management.Test/Storage.Management.Test.csproj index 4525a5e2093c..0f5dfd834093 100644 --- a/src/Storage/Storage.Management.Test/Storage.Management.Test.csproj +++ b/src/Storage/Storage.Management.Test/Storage.Management.Test.csproj @@ -11,10 +11,10 @@ - - - - + + + + diff --git a/src/Storage/Storage.Management/Az.Storage.psd1 b/src/Storage/Storage.Management/Az.Storage.psd1 index 208a461de649..a3ef78085f20 100644 --- a/src/Storage/Storage.Management/Az.Storage.psd1 +++ b/src/Storage/Storage.Management/Az.Storage.psd1 @@ -165,7 +165,8 @@ CmdletsToExport = 'Get-AzStorageAccount', 'Get-AzStorageAccountKey', 'New-AzDataLakeGen2Item', 'Move-AzDataLakeGen2Item', 'Remove-AzDataLakeGen2Item', 'Update-AzDataLakeGen2Item', 'Set-AzDataLakeGen2ItemAclObject', 'Get-AzDataLakeGen2ItemContent', - 'Invoke-AzStorageAccountFailover' + 'Invoke-AzStorageAccountFailover', + 'Get-AzStorageBlobQueryResult', 'New-AzStorageBlobQueryConfig' # Variables to export from this module # VariablesToExport = @() diff --git a/src/Storage/Storage.Management/ChangeLog.md b/src/Storage/Storage.Management/ChangeLog.md index a87c17d196d6..6b85d8594f62 100644 --- a/src/Storage/Storage.Management/ChangeLog.md +++ b/src/Storage/Storage.Management/ChangeLog.md @@ -18,6 +18,9 @@ - Additional information about change #1 --> ## Upcoming Release +* Supported blob query acceleration + - `Get-AzStorageBlobQueryResult` + - `New-AzStorageBlobQueryConfig` ## Version 2.4.0 * Supported create container/blob Sas token with new permission x,t diff --git a/src/Storage/Storage.Management/help/Az.Storage.md b/src/Storage/Storage.Management/help/Az.Storage.md index 2c2e45f194b1..8ce12708b9b8 100644 --- a/src/Storage/Storage.Management/help/Az.Storage.md +++ b/src/Storage/Storage.Management/help/Az.Storage.md @@ -83,6 +83,9 @@ Downloads a storage blob. ### [Get-AzStorageBlobCopyState](Get-AzStorageBlobCopyState.md) Gets the copy status of an Azure Storage blob. +### [Get-AzStorageBlobQueryResult](Get-AzStorageBlobQueryResult.md) +Applies a simple Structured Query Language (SQL) statement on a blob's contents and save only the queried subset of the data to a local file. + ### [Get-AzStorageBlobServiceProperty](Get-AzStorageBlobServiceProperty.md) Gets service properties for Azure Storage Blob services. @@ -170,6 +173,9 @@ Creates a ManagementPolicy rule object, which can be used in Set-AzStorageAccoun ### [New-AzStorageAccountSASToken](New-AzStorageAccountSASToken.md) Creates an account-level SAS token. +### [New-AzStorageBlobQueryConfig](New-AzStorageBlobQueryConfig.md) +Creates a blob query configuration object, which can be used in Get-AzStorageBlobQueryResult. + ### [New-AzStorageBlobSASToken](New-AzStorageBlobSASToken.md) Generates a SAS token for an Azure storage blob. diff --git a/src/Storage/Storage.Management/help/Get-AzStorageBlobQueryResult.md b/src/Storage/Storage.Management/help/Get-AzStorageBlobQueryResult.md new file mode 100644 index 000000000000..f83c567f7277 --- /dev/null +++ b/src/Storage/Storage.Management/help/Get-AzStorageBlobQueryResult.md @@ -0,0 +1,391 @@ +--- +external help file: Microsoft.Azure.PowerShell.Cmdlets.Storage.dll-Help.xml +Module Name: Az.Storage +online version: https://docs.microsoft.com/en-us/powershell/module/Az.storage/get-azstorageblobqueryresult +schema: 2.0.0 +--- + +# Get-AzStorageBlobQueryResult + +## SYNOPSIS +Applies a simple Structured Query Language (SQL) statement on a blob's contents and save only the queried subset of the data to a local file. + +## SYNTAX + +### NamePipeline (Default) +``` +Get-AzStorageBlobQueryResult [-Blob] [-Container] [-SnapshotTime ] + [-VersionId ] -QueryString -ResultFile + [-InputTextConfiguration ] + [-OutputTextConfiguration ] [-PassThru] [-Context ] + [-ServerTimeoutPerRequest ] [-ClientTimeoutPerRequest ] + [-DefaultProfile ] [-ConcurrentTaskCount ] [-WhatIf] [-Confirm] + [] +``` + +### BlobPipeline +``` +Get-AzStorageBlobQueryResult -BlobBaseClient -QueryString -ResultFile + [-InputTextConfiguration ] + [-OutputTextConfiguration ] [-PassThru] [-Context ] + [-ServerTimeoutPerRequest ] [-ClientTimeoutPerRequest ] + [-DefaultProfile ] [-ConcurrentTaskCount ] [-WhatIf] [-Confirm] + [] +``` + +### ContainerPipeline +``` +Get-AzStorageBlobQueryResult -BlobContainerClient [-Blob] + [-SnapshotTime ] [-VersionId ] -QueryString -ResultFile + [-InputTextConfiguration ] + [-OutputTextConfiguration ] [-PassThru] [-Context ] + [-ServerTimeoutPerRequest ] [-ClientTimeoutPerRequest ] + [-DefaultProfile ] [-ConcurrentTaskCount ] [-WhatIf] [-Confirm] + [] +``` + +## DESCRIPTION +The **Get-AzStorageBlobQueryResult** cmdlet applies a simple Structured Query Language (SQL) statement on a blob's contents and save the queried subset of the data to a local file. + +## EXAMPLES + +### Example 1: Query a blob +```powershell +PS C:\> $inputconfig = New-AzStorageBlobQueryConfig -AsCsv -HasHeader + +PS C:\> $outputconfig = New-AzStorageBlobQueryConfig -AsJson + +PS C:\> $queryString = "SELECT * FROM BlobStorage WHERE Name = 'a'" + +PS C:\> $result = Get-AzStorageBlobQueryResult -Container $containerName -Blob $blobName -QueryString $queryString -ResultFile "c:\resultfile.json" -InputTextConfiguration $inputconfig -OutputTextConfiguration $outputconfig -Context $ctx + +PS C:\> $result + +BytesScanned FailureCount BlobQueryError +------------ ------------ -------------- + 449 0 +``` + +This command querys a blob succsssfully with input config as csv, and output config as json, and save the output to local file "c:\resultfile.json". + +### Example 2: Query a blob snapshot +```powershell +PS C:\> $blob = Get-AzStorageBlob -Container $containerName -Blob $blobName -SnapshotTime "2020-07-29T11:08:21.1097874Z" -Context $ctx + +PS C:\> $inputconfig = New-AzStorageBlobQueryConfig -AsCsv -ColumnSeparator "," -QuotationCharacter """" -EscapeCharacter "\" -RecordSeparator "`n" -HasHeader + +PS C:\> $outputconfig = New-AzStorageBlobQueryConfig -AsJson -RecordSeparator "`n" + +PS C:\> $queryString = "SELECT * FROM BlobStorage WHERE _1 LIKE '1%%'" + +PS C:\> $result = $blob | Get-AzStorageBlobQueryResult -QueryString $queryString -ResultFile $localFilePath -InputTextConfiguration $inputconfig -OutputTextConfiguration $outputconfig + +PS C:\> $result + +BytesScanned FailureCount BlobQueryError +------------ ------------ -------------- + 187064971 1 {ParseError} + + + +PS C:\> $result.BlobQueryError + +Name Description IsFatal Position +---- ----------- ------- -------- +ParseError Unexpected token '1' at [byte: 3077737]. Expecting token ','. True 7270632 +``` + +This command first gets a blob object for blob snapshot, then queries the blob snapshot and show the result include query error. + +## PARAMETERS + +### -Blob +Blob name + +```yaml +Type: System.String +Parameter Sets: NamePipeline, ContainerPipeline +Aliases: + +Required: True +Position: 0 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -BlobBaseClient +BlobBaseClient Object + +```yaml +Type: Azure.Storage.Blobs.Specialized.BlobBaseClient +Parameter Sets: BlobPipeline +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName) +Accept wildcard characters: False +``` + +### -BlobContainerClient +BlobContainerClient Object + +```yaml +Type: Azure.Storage.Blobs.BlobContainerClient +Parameter Sets: ContainerPipeline +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName) +Accept wildcard characters: False +``` + +### -ClientTimeoutPerRequest +The client side maximum execution time for each request in seconds. + +```yaml +Type: System.Nullable`1[System.Int32] +Parameter Sets: (All) +Aliases: ClientTimeoutPerRequestInSeconds + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ConcurrentTaskCount +The total amount of concurrent async tasks. +The default value is 10. + +```yaml +Type: System.Nullable`1[System.Int32] +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Container +Container name + +```yaml +Type: System.String +Parameter Sets: NamePipeline +Aliases: + +Required: True +Position: 1 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Context +Azure Storage Context Object + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName, ByValue) +Accept wildcard characters: False +``` + +### -DefaultProfile +The credentials, account, tenant, and subscription used for communication with Azure. + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.Core.IAzureContextContainer +Parameter Sets: (All) +Aliases: AzureRmContext, AzureCredential + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -InputTextConfiguration +The configuration used to handled the query input text. Create configuration object the with New-AzStorageBlobQueryConfig. + +```yaml +Type: Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.PSBlobQueryTextConfiguration +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -OutputTextConfiguration +The configuration used to handled the query output text. Create configuration object the with New-AzStorageBlobQueryConfig. + +```yaml +Type: Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.PSBlobQueryTextConfiguration +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -PassThru +Return whether the specified blob is successfully queried. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -QueryString +Query string, see more details in: https://docs.microsoft.com/en-us/azure/storage/blobs/query-acceleration-sql-reference + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ResultFile +Local file path to save the query result. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ServerTimeoutPerRequest +The server time out for each request in seconds. + +```yaml +Type: System.Nullable`1[System.Int32] +Parameter Sets: (All) +Aliases: ServerTimeoutPerRequestInSeconds + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -SnapshotTime +Blob SnapshotTime + +```yaml +Type: System.Nullable`1[System.DateTimeOffset] +Parameter Sets: NamePipeline, ContainerPipeline +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -VersionId +Blob VersionId + +```yaml +Type: System.String +Parameter Sets: NamePipeline, ContainerPipeline +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -WhatIf +Shows what would happen if the cmdlet runs. +The cmdlet is not run. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +### Azure.Storage.Blobs.Specialized.BlobBaseClient + +### Azure.Storage.Blobs.BlobContainerClient + +### Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext + +## OUTPUTS + +### System.Boolean + +## NOTES + +## RELATED LINKS diff --git a/src/Storage/Storage.Management/help/New-AzStorageBlobQueryConfig.md b/src/Storage/Storage.Management/help/New-AzStorageBlobQueryConfig.md new file mode 100644 index 000000000000..791a4dd2a612 --- /dev/null +++ b/src/Storage/Storage.Management/help/New-AzStorageBlobQueryConfig.md @@ -0,0 +1,190 @@ +--- +external help file: Microsoft.Azure.PowerShell.Cmdlets.Storage.dll-Help.xml +Module Name: Az.Storage +online version: https://docs.microsoft.com/en-us/powershell/module/Az.storage/new-azstorageblobqueryconfig +schema: 2.0.0 +--- + +# New-AzStorageBlobQueryConfig + +## SYNOPSIS +Creates a blob query configuration object, which can be used in Get-AzStorageBlobQueryResult. + +## SYNTAX + +### Csv (Default) +``` +New-AzStorageBlobQueryConfig [-AsCsv] [-RecordSeparator ] [-ColumnSeparator ] + [-QuotationCharacter ] [-EscapeCharacter ] [-HasHeader] [-AsJob] [] +``` + +### Json +``` +New-AzStorageBlobQueryConfig [-AsJson] [-RecordSeparator ] [-AsJob] [] +``` + +## DESCRIPTION +The **New-AzStorageBlobQueryConfig** cmdlet creates a blob query configuration object, which can be used in Get-AzStorageBlobQueryResult. + +## EXAMPLES + +### Example 1: Create blob query configures , and query a blob +```powershell +PS C:\> $inputconfig = New-AzStorageBlobQueryConfig -AsCsv -ColumnSeparator "," -QuotationCharacter """" -EscapeCharacter "\" -RecordSeparator "`n" -HasHeader + +PS C:\> $outputconfig = New-AzStorageBlobQueryConfig -AsJson -RecordSeparator "`n" + +PS C:\> $queryString = "SELECT * FROM BlobStorage WHERE Name = 'a'" + +PS C:\> $result = Get-AzStorageBlobQueryResult -Container $containerName -Blob $blobName -QueryString $queryString -ResultFile "c:\resultfile.json" -InputTextConfiguration $inputconfig -OutputTextConfiguration $outputconfig -Context $ctx + +PS C:\> $result + +BytesScanned FailureCount BlobQueryError +------------ ------------ -------------- + 449 0 +``` + +This command first create input configuration object as csv, and output configuration object as json, then use the 2 configurations to query blob. + +## PARAMETERS + +### -AsCsv +Indicate to create a Blob Query Configuration for CSV. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: Csv +Aliases: + +Required: True +Position: 0 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -AsJob +Run cmdlet in the background + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -AsJson +Indicate to create a Blob Query Configuration for Json. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: Json +Aliases: + +Required: True +Position: 0 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ColumnSeparator +Optional. +The string used to separate columns. + +```yaml +Type: System.String +Parameter Sets: Csv +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -EscapeCharacter +Optional. +The char used as an escape character. + +```yaml +Type: System.Nullable`1[System.Char] +Parameter Sets: Csv +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -HasHeader +Optional. +Indicate it represent the data has headers. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: Csv +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -QuotationCharacter +Optional. +The char used to quote a specific field. + +```yaml +Type: System.Char +Parameter Sets: Csv +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -RecordSeparator +Optional. +The string used to separate records. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +### None + +## OUTPUTS + +### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.PSBlobQueryTextConfiguration + +## NOTES + +## RELATED LINKS diff --git a/src/Storage/Storage.Test/Service/MockStorageBlobManagement.cs b/src/Storage/Storage.Test/Service/MockStorageBlobManagement.cs index f8ffdf3d04bd..b9b0b38e3a84 100644 --- a/src/Storage/Storage.Test/Service/MockStorageBlobManagement.cs +++ b/src/Storage/Storage.Test/Service/MockStorageBlobManagement.cs @@ -24,6 +24,7 @@ using Microsoft.Azure.Storage.Auth; using Microsoft.Azure.Storage.Blob; using Microsoft.Azure.Storage.Shared.Protocol; +using Azure.Storage.Blobs; namespace Microsoft.WindowsAzure.Commands.Storage.Test.Service { @@ -731,6 +732,16 @@ public Task StartCopyAsync(CloudBlob blob, Uri source, StandardBlobTier? throw new NotImplementedException(); } + public BlobContainerClient GetBlobContainerClient(string name, BlobClientOptions options = null) + { + throw new NotImplementedException(); + } + + public BlobServiceClient GetBlobServiceClient(BlobClientOptions options = null) + { + throw new NotImplementedException(); + } + /// /// The storage context /// diff --git a/src/Storage/Storage/Blob/Cmdlet/GetAzStorageBlobQueryResult.cs b/src/Storage/Storage/Blob/Cmdlet/GetAzStorageBlobQueryResult.cs new file mode 100644 index 000000000000..30cb355aa25a --- /dev/null +++ b/src/Storage/Storage/Blob/Cmdlet/GetAzStorageBlobQueryResult.cs @@ -0,0 +1,208 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Storage.Blob +{ + using global::Azure.Storage.Blobs; + using global::Azure.Storage.Blobs.Specialized; + using Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel; + using Microsoft.WindowsAzure.Commands.Storage.Common; + using Microsoft.WindowsAzure.Commands.Storage.Model.Contract; + using System; + using System.Collections.Generic; + using System.IO; + using System.Management.Automation; + using System.Security.Permissions; + using System.Threading.Tasks; + using Track2Models = global::Azure.Storage.Blobs.Models; + + [Cmdlet("Get", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "StorageBlobQueryResult", DefaultParameterSetName = NameParameterSet, SupportsShouldProcess = true),OutputType(typeof(BlobQueryOutput))] + public class GetStorageAzureBlobQueryResultCommand : StorageCloudBlobCmdletBase + { + /// + /// Blob Pipeline parameter set name + /// + private const string BlobPipelineParameterSet = "BlobPipeline"; + + /// + /// container pipeline paremeter set name + /// + private const string ContainerPipelineParameterSet = "ContainerPipeline"; + + /// + /// blob name and container name parameter set + /// + private const string NameParameterSet = "NamePipeline"; + + private List queryErrors = new List(); + private long bytesScanned = 0; + + [Parameter(HelpMessage = "BlobBaseClient Object", Mandatory = true, + ValueFromPipelineByPropertyName = true, ParameterSetName = BlobPipelineParameterSet)] + [ValidateNotNull] + public BlobBaseClient BlobBaseClient { get; set; } + + [Parameter(HelpMessage = "BlobContainerClient Object", Mandatory = true, + ValueFromPipelineByPropertyName = true, ParameterSetName = ContainerPipelineParameterSet)] + public BlobContainerClient BlobContainerClient { get; set; } + + [Parameter(ParameterSetName = ContainerPipelineParameterSet, Mandatory = true, Position = 0, HelpMessage = "Blob name")] + [Parameter(ParameterSetName = NameParameterSet, Mandatory = true, Position = 0, HelpMessage = "Blob name")] + public string Blob + { + get { return BlobName; } + set { BlobName = value; } + } + private string BlobName = String.Empty; + + [Parameter(HelpMessage = "Container name", Mandatory = true, Position = 1, + ParameterSetName = NameParameterSet)] + [ValidateNotNullOrEmpty] + public string Container + { + get { return ContainerName; } + set { ContainerName = value; } + } + private string ContainerName = String.Empty; + + [Parameter(HelpMessage = "Blob SnapshotTime", Mandatory = false, ParameterSetName = ContainerPipelineParameterSet)] + [Parameter(HelpMessage = "Blob SnapshotTime", Mandatory = false, ParameterSetName = NameParameterSet)] + [ValidateNotNullOrEmpty] + public DateTimeOffset? SnapshotTime { get; set; } + + [Parameter(HelpMessage = "Blob VersionId", Mandatory = false, ParameterSetName = ContainerPipelineParameterSet)] + [Parameter(HelpMessage = "Blob VersionId", Mandatory = false, ParameterSetName = NameParameterSet)] + [ValidateNotNullOrEmpty] + public string VersionId { get; set; } + + [Parameter(HelpMessage = "Query string, see more details in: https://docs.microsoft.com/en-us/azure/storage/blobs/query-acceleration-sql-reference", Mandatory = true)] + [ValidateNotNullOrEmpty] + public string QueryString { get; set; } + + [Parameter(HelpMessage = "Local file path to save the query result.", Mandatory = true)] + [ValidateNotNullOrEmpty] + public string ResultFile { get; set; } + + [Parameter(HelpMessage = "The configuration used to handled the query input text.", Mandatory = false)] + [ValidateNotNullOrEmpty] + public PSBlobQueryTextConfiguration InputTextConfiguration { get; set; } + + [Parameter(HelpMessage = "The configuration used to handled the query output text.", Mandatory = false)] + [ValidateNotNullOrEmpty] + public PSBlobQueryTextConfiguration OutputTextConfiguration { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Return whether the specified blob is successfully queried.")] + public SwitchParameter PassThru { get; set; } + + protected override bool UseTrack2Sdk() + { + return true; + } + + /// + /// Initializes a new instance of the RemoveStorageAzureBlobCommand class. + /// + public GetStorageAzureBlobQueryResultCommand() + : this(null) + { + } + + /// + /// Initializes a new instance of the RemoveStorageAzureBlobCommand class. + /// + /// IStorageBlobManagement channel + public GetStorageAzureBlobQueryResultCommand(IStorageBlobManagement channel) + { + Channel = channel; + } + + + /// + /// Cmdlet begin processing + /// + protected override void BeginProcessing() + { + base.BeginProcessing(); + OutputStream.ConfirmWriter = (s1, s2, s3) => ShouldContinue(s2, s3); + } + + /// + /// execute command + /// + [PermissionSet(SecurityAction.Demand, Name = "FullTrust")] + public override void ExecuteCmdlet() + { + Func taskGenerator = null; + IStorageBlobManagement localChannel = Channel; + + switch (ParameterSetName) + { + case BlobPipelineParameterSet: + break; + case ContainerPipelineParameterSet: + this.BlobBaseClient = Util.GetTrack2BlobClient(this.BlobContainerClient, + this.Blob, Channel.StorageContext, + this.VersionId, + null, + this.SnapshotTime is null? null : this.SnapshotTime.Value.ToString("o"), + this.ClientOptions, Track2Models.BlobType.Block); + break; + case NameParameterSet: + default: + BlobContainerClient container = localChannel.GetBlobContainerClient(this.Container, this.ClientOptions); + this.BlobBaseClient = Util.GetTrack2BlobClient( + container, + this.Blob, Channel.StorageContext, + this.VersionId, + null, + this.SnapshotTime is null ? null : this.SnapshotTime.Value.ToString("o"), + this.ClientOptions, Track2Models.BlobType.Block); + break; + } + + taskGenerator = (taskId) => QueryAzureBlob(taskId, localChannel, this.BlobBaseClient, this.QueryString, true); + RunTask(taskGenerator); + } + + internal async Task QueryAzureBlob(long taskId, IStorageBlobManagement localChannel, BlobBaseClient blob, string query, bool headers) + { + IProgress progressHandler = new Progress((finishedBytes) => + { + bytesScanned = finishedBytes; + }); + + // preapre query Option + // Not show the ProgressHandler now, since the ProgressHandler can't represent the read query progress + Track2Models.BlobQueryOptions queryOption = new Track2Models.BlobQueryOptions + { + InputTextConfiguration = this.InputTextConfiguration is null ? null : this.InputTextConfiguration.ParseBlobQueryTextConfiguration(), + OutputTextConfiguration = this.OutputTextConfiguration is null ? null : this.OutputTextConfiguration.ParseBlobQueryTextConfiguration(), + ProgressHandler = progressHandler, + }; + + queryOption.ErrorHandler += (e) => + { + queryErrors.Add(new PSBlobQueryError(e)); + }; + + using (var reader = (await ((BlockBlobClient)blob).QueryAsync(query, queryOption, CmdletCancellationToken)).Value.Content) + { + FileStream fs = File.Create(this.ResultFile); + reader.CopyTo(fs); + fs.Close(); + } + OutputStream.WriteObject(taskId, new BlobQueryOutput(bytesScanned, queryErrors)); + } + } +} diff --git a/src/Storage/Storage/Blob/Cmdlet/NewAzStorageBlobQueryConfig .cs b/src/Storage/Storage/Blob/Cmdlet/NewAzStorageBlobQueryConfig .cs new file mode 100644 index 000000000000..cd2f27e42e93 --- /dev/null +++ b/src/Storage/Storage/Blob/Cmdlet/NewAzStorageBlobQueryConfig .cs @@ -0,0 +1,132 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using Microsoft.WindowsAzure.Commands.Common; +using Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel; +using Microsoft.WindowsAzure.Commands.Storage; +using System.Management.Automation; + +namespace Microsoft.Azure.Commands.Management.Storage +{ + [Cmdlet("New", ResourceManager.Common.AzureRMConstants.AzureRMPrefix + "StorageBlobQueryConfig", DefaultParameterSetName = CsvParameterSet), OutputType(typeof(PSBlobQueryTextConfiguration))] + public class NewAzStorageBlobQueryConfigCommand : AzureDataCmdlet + { + /// + /// default parameter set name + /// + private const string CsvParameterSet = "Csv"; + + /// + /// Json parameter set name + /// + private const string JsonParameterSet = "Json"; + + [Parameter(Mandatory = true, + Position = 0, + HelpMessage = "Indicate to create a Blob Query Configuration for CSV.", + ParameterSetName = CsvParameterSet)] + [ValidateNotNullOrEmpty] + public SwitchParameter AsCsv { get; set; } + + [Parameter(Mandatory = true, + Position = 0, + HelpMessage = "Indicate to create a Blob Query Configuration for Json.", + ParameterSetName = JsonParameterSet)] + [ValidateNotNullOrEmpty] + public SwitchParameter AsJson { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Optional. The string used to separate records.", ParameterSetName = CsvParameterSet)] + [Parameter(Mandatory = false, HelpMessage = "Optional. The string used to separate records.", ParameterSetName = JsonParameterSet)] + public string RecordSeparator { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Optional. The string used to separate columns.", ParameterSetName = CsvParameterSet)] + public string ColumnSeparator { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Optional. The char used to quote a specific field.", ParameterSetName = CsvParameterSet)] + public char QuotationCharacter + { + get + { + if (quotationCharacter != null) + { + return quotationCharacter.Value; + } + else + { + return '\0'; + } + } + set + { + quotationCharacter = value; + } + } + public char? quotationCharacter = null; + + [Parameter(Mandatory = false, HelpMessage = "Optional. The char used as an escape character.", ParameterSetName = CsvParameterSet)] + public char? EscapeCharacter + { + get + { + if (escapeCharacter != null) + { + return escapeCharacter.Value; + } + else + { + return '\0'; + } + } + set + { + escapeCharacter = value; + } + } + public char? escapeCharacter = null; + + [Parameter(Mandatory = false, HelpMessage = "Optional. Indicate it represent the data has headers.", ParameterSetName = CsvParameterSet)] + public SwitchParameter HasHeader { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Run cmdlet in the background")] + public virtual SwitchParameter AsJob { get; set; } + + public override void ExecuteCmdlet() + { + base.ExecuteCmdlet(); + + if (this.AsJson.IsPresent) //Json + { + PSBlobQueryTextConfiguration queryConfig = new PSBlobQueryJsonTextConfiguration() + { + RecordSeparator = this.RecordSeparator, + Type = BlobQueryConfigType.Json + }; + WriteObject(queryConfig); + } + else // Csv + { + PSBlobQueryTextConfiguration queryConfig = new PSBlobQueryCsvTextConfiguration() + { + RecordSeparator = this.RecordSeparator, + ColumnSeparator = this.ColumnSeparator, + QuotationCharacter = this.quotationCharacter, + EscapeCharacter = this.escapeCharacter, + HasHeaders = this.HasHeader.IsPresent, + Type = BlobQueryConfigType.Csv + }; + WriteObject(queryConfig); + } + } + } +} diff --git a/src/Storage/Storage/Common/BlobQueryTextConfiguration.cs b/src/Storage/Storage/Common/BlobQueryTextConfiguration.cs new file mode 100644 index 000000000000..41950c34f5ee --- /dev/null +++ b/src/Storage/Storage/Common/BlobQueryTextConfiguration.cs @@ -0,0 +1,156 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using Azure.Storage.Blobs.Models; +using System.Collections.Generic; + +namespace Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel +{ + /// + /// Enum to define the type of BlobQueryConfig + /// + public enum BlobQueryConfigType + { + Csv, + Json + } + + /// + /// Wrapper Class for PSBlobQueryTextConfiguration, BlobQueryCsvTextConfiguration, BlobQueryJsonTextConfiguration + /// + public abstract class PSBlobQueryTextConfiguration + { + public BlobQueryConfigType Type { get; set; } + public string RecordSeparator { get; set; } + + public BlobQueryTextConfiguration ParseBlobQueryTextConfiguration() + { + if (this.Type == BlobQueryConfigType.Csv) //csv + { + PSBlobQueryCsvTextConfiguration csvconfig = (PSBlobQueryCsvTextConfiguration)this; + return new BlobQueryCsvTextConfiguration() + { + RecordSeparator = csvconfig.RecordSeparator, + ColumnSeparator = csvconfig.ColumnSeparator, + QuotationCharacter = csvconfig.QuotationCharacter, + EscapeCharacter = csvconfig.EscapeCharacter, + HasHeaders = csvconfig.HasHeaders + }; + } + else //json + { + PSBlobQueryJsonTextConfiguration jsonconfig = (PSBlobQueryJsonTextConfiguration)this; + return new BlobQueryJsonTextConfiguration() + { + RecordSeparator = jsonconfig.RecordSeparator + }; + } + } + } + + /// + /// Wrapper Class for BlobQueryJsonTextConfiguration + /// + public class PSBlobQueryJsonTextConfiguration : PSBlobQueryTextConfiguration + { + public PSBlobQueryJsonTextConfiguration() { } + + public PSBlobQueryJsonTextConfiguration(BlobQueryJsonTextConfiguration config) + { + this.RecordSeparator = config.RecordSeparator; + this.Type = BlobQueryConfigType.Json; + } + + public BlobQueryJsonTextConfiguration ParseBlobQueryJsonTextConfiguration() + { + return new BlobQueryJsonTextConfiguration() + { + RecordSeparator = this.RecordSeparator + }; + } + } + + /// + /// Wrapper Class for BlobQueryCsvTextConfiguration + /// + public class PSBlobQueryCsvTextConfiguration : PSBlobQueryTextConfiguration + { + public string ColumnSeparator { get; set; } + public char? QuotationCharacter { get; set; } + public char? EscapeCharacter { get; set; } + public bool HasHeaders { get; set; } + + + public PSBlobQueryCsvTextConfiguration() { } + + public PSBlobQueryCsvTextConfiguration(BlobQueryCsvTextConfiguration config) + { + this.RecordSeparator = config.RecordSeparator; + this.ColumnSeparator = config.ColumnSeparator; + this.QuotationCharacter = config.QuotationCharacter; + this.EscapeCharacter = config.EscapeCharacter; + this.HasHeaders = config.HasHeaders; + this.Type = BlobQueryConfigType.Csv; + } + + public BlobQueryCsvTextConfiguration ParseBlobQueryCsvTextConfiguration() + { + return new BlobQueryCsvTextConfiguration() + { + RecordSeparator = this.RecordSeparator, + ColumnSeparator = this.ColumnSeparator, + QuotationCharacter = this.QuotationCharacter, + EscapeCharacter = this.EscapeCharacter, + HasHeaders = this.HasHeaders + }; + } + } + + /// + /// Wrapper of BlobQueryError + /// + public class PSBlobQueryError + { + public string Name { get; } + public string Description { get; } + public bool IsFatal { get; } + public long Position { get; } + + public PSBlobQueryError (BlobQueryError error) + { + this.Name = error.Name; + this.Description = error.Description; + this.IsFatal = error.IsFatal; + this.Position = error.Position; + } + } + + /// + /// Used to output the query result, which include scanned bytes and errors + /// + public class BlobQueryOutput + { + public long BytesScanned { get; set; } + public int FailureCount { get; set; } + public PSBlobQueryError[] BlobQueryError { get; set; } + + public BlobQueryOutput() { } + public BlobQueryOutput(long bytesScanned, List errors) + { + this.BytesScanned = bytesScanned; + this.FailureCount = errors is null ? 0 : errors.Count; + this.BlobQueryError = (errors is null || errors.Count == 0) ? null : errors.ToArray(); + } + } +} diff --git a/src/Storage/Storage/Model/Contract/IStorageBlobManagement.cs b/src/Storage/Storage/Model/Contract/IStorageBlobManagement.cs index 8a6a5f5c1844..0c0aeb9518ad 100644 --- a/src/Storage/Storage/Model/Contract/IStorageBlobManagement.cs +++ b/src/Storage/Storage/Model/Contract/IStorageBlobManagement.cs @@ -23,6 +23,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Model.Contract using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; + using global::Azure.Storage.Blobs; /// /// Blob management interface @@ -76,6 +77,20 @@ public interface IStorageBlobManagement : IStorageManagement /// A CloudBlobContainer in local memory CloudBlobContainer GetContainerReference(String name); + /// + /// Get an BlobContainerClient instance in local + /// + /// Container name + /// A BlobContainerClient in local memory + BlobContainerClient GetBlobContainerClient(string name, BlobClientOptions options = null); + + /// + /// Get an BlobServiceClient instance in local + /// + /// Container name + /// A BlobServiceClient in local memory + BlobServiceClient GetBlobServiceClient(BlobClientOptions options = null); + /// /// Get blob reference with properties and meta data from server /// diff --git a/src/Storage/Storage/Model/Contract/StorageBlobManagement.cs b/src/Storage/Storage/Model/Contract/StorageBlobManagement.cs index 921f91dffd28..d8970709860b 100644 --- a/src/Storage/Storage/Model/Contract/StorageBlobManagement.cs +++ b/src/Storage/Storage/Model/Contract/StorageBlobManagement.cs @@ -29,6 +29,8 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Model.Contract using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; + using global::Azure.Storage.Blobs; + using global::Azure.Storage; /// /// Blob management @@ -144,6 +146,43 @@ public CloudBlobContainer GetContainerReference(string name) return this.BlobClient.GetContainerReference(name); } + /// + /// Get an BlobContainerClient instance in local + /// + /// Container name + /// A BlobContainerClient in local memory + public BlobContainerClient GetBlobContainerClient(string name, BlobClientOptions options = null) + { + return GetBlobServiceClient(options).GetBlobContainerClient(name); + } + + /// + /// Get an BlobServiceClient instance in local + /// + /// Container name + /// A BlobServiceClient in local memory + public BlobServiceClient GetBlobServiceClient(BlobClientOptions options = null) + { + if (blobServiceClient == null) + { + if (this.StorageContext.StorageAccount.Credentials.IsToken) //Oauth + { + blobServiceClient = new BlobServiceClient(this.StorageContext.StorageAccount.BlobEndpoint, this.StorageContext.Track2OauthToken, options); + } + else if (this.StorageContext.StorageAccount.Credentials.IsSharedKey) //Shared Key + { + blobServiceClient = new BlobServiceClient(this.StorageContext.StorageAccount.BlobEndpoint, + new StorageSharedKeyCredential(this.StorageContext.StorageAccountName, this.StorageContext.StorageAccount.Credentials.ExportBase64EncodedKey()), options); + } + else //sas, Anonymous + { + blobServiceClient = new BlobServiceClient(this.StorageContext.StorageAccount.BlobEndpoint, options); + } + } + return blobServiceClient; + } + private BlobServiceClient blobServiceClient = null; + /// /// Create the container if not exists /// diff --git a/src/Storage/Storage/Storage.csproj b/src/Storage/Storage/Storage.csproj index d25985bda03d..8ee2047c1849 100644 --- a/src/Storage/Storage/Storage.csproj +++ b/src/Storage/Storage/Storage.csproj @@ -12,10 +12,10 @@ - - - - + + + + diff --git a/src/lib/NetCorePreloadAssemblies/Azure.Core.dll b/src/lib/NetCorePreloadAssemblies/Azure.Core.dll index fba773755864..b10773c3d9c6 100644 Binary files a/src/lib/NetCorePreloadAssemblies/Azure.Core.dll and b/src/lib/NetCorePreloadAssemblies/Azure.Core.dll differ diff --git a/src/lib/NetFxPreloadAssemblies/Azure.Core.dll b/src/lib/NetFxPreloadAssemblies/Azure.Core.dll index fba773755864..b10773c3d9c6 100644 Binary files a/src/lib/NetFxPreloadAssemblies/Azure.Core.dll and b/src/lib/NetFxPreloadAssemblies/Azure.Core.dll differ diff --git a/tools/StaticAnalysis/Exceptions/Az.Storage/SignatureIssues.csv b/tools/StaticAnalysis/Exceptions/Az.Storage/SignatureIssues.csv index 4b61c895b28e..99c5cc82844d 100644 --- a/tools/StaticAnalysis/Exceptions/Az.Storage/SignatureIssues.csv +++ b/tools/StaticAnalysis/Exceptions/Az.Storage/SignatureIssues.csv @@ -63,4 +63,5 @@ "Microsoft.Azure.PowerShell.Cmdlets.Storage.Management.dll","Microsoft.Azure.Commands.Management.Storage.RevokeAzureStorageAccountUserDelegationKeysCommand","Revoke-AzStorageAccountUserDelegationKeys","1","8400","Revoke-AzStorageAccountUserDelegationKeys uses the noun 'AzStorageAccountUserDelegationKeys', which does not follow the enforced naming convention of using a singular noun for a cmdlet name.","Consider using a singular noun for the cmdlet name." "Microsoft.Azure.PowerShell.Cmdlets.Storage.dll","Microsoft.Azure.Commands.Management.Storage.NewAzDataLakeGen2ItemAclObjectCommand","New-AzDataLakeGen2ItemAclObject","1","8100","New-AzDataLakeGen2ItemAclObject Does not support ShouldProcess but the cmdlet verb New indicates that it should.","Determine if the cmdlet should implement ShouldProcess and if so determine if it should implement Force / ShouldContinue" "Microsoft.Azure.PowerShell.Cmdlets.Storage.Management.dll","Microsoft.Azure.Commands.Management.Storage.GetAzureStorageAccountCommand","Get-AzStorageAccount","1","8410","Parameter IncludeGeoReplicationStats of cmdlet Get-AzStorageAccount does not follow the enforced naming convention of using a singular noun for a parameter name.","Consider using a singular noun for the parameter name." -"Microsoft.Azure.PowerShell.Cmdlets.Storage.dll","Microsoft.Azure.Commands.Management.Storage.SetAzDataLakeGen2ItemAclObjectCommand","Set-AzDataLakeGen2ItemAclObject","1","8100","Set-AzDataLakeGen2ItemAclObject Does not support ShouldProcess but the cmdlet verb Set indicates that it should.","Determine if the cmdlet should implement ShouldProcess and if so determine if it should implement Force / ShouldContinue" \ No newline at end of file +"Microsoft.Azure.PowerShell.Cmdlets.Storage.dll","Microsoft.Azure.Commands.Management.Storage.SetAzDataLakeGen2ItemAclObjectCommand","Set-AzDataLakeGen2ItemAclObject","1","8100","Set-AzDataLakeGen2ItemAclObject Does not support ShouldProcess but the cmdlet verb Set indicates that it should.","Determine if the cmdlet should implement ShouldProcess and if so determine if it should implement Force / ShouldContinue" +"Microsoft.Azure.PowerShell.Cmdlets.Storage.dll","Microsoft.Azure.Commands.Management.Storage.NewAzStorageBlobQueryConfigCommand","New-AzStorageBlobQueryConfig","1","8100","New-AzStorageBlobQueryConfig Does not support ShouldProcess but the cmdlet verb New indicates that it should.","Determine if the cmdlet should implement ShouldProcess and if so determine if it should implement Force / ShouldContinue" \ No newline at end of file