Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

azurerm_data_factory_dataset_parquet - Support azure_blob_fs_location #23261

Merged
merged 6 commits into from
Sep 20, 2023
Merged

azurerm_data_factory_dataset_parquet - Support azure_blob_fs_location #23261

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f88432c
`data_factory_dataset_parquet` support `azure_blob_fs_location`
bianyifan Sep 14, 2023
607973a
support read azure_blob_fs_location
bianyifan Sep 14, 2023
2dbefda
add acc test
bianyifan Sep 14, 2023
37811b3
ExactlyOneOf and doc
bianyifan Sep 19, 2023
6a277e5
Merge remote-tracking branch 'remotes/upstream/main' into data-factor…
bianyifan Sep 19, 2023
314baa1
also update azure_blob_fs_location for dataset_delimited_text
bianyifan Sep 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
42 changes: 16 additions & 26 deletions internal/services/datafactory/data_factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,18 +424,14 @@ func expandDataFactoryDatasetAzureBlobFSLocation(d *pluginsdk.ResourceData) data

props := azureBlobFsLocations[0].(map[string]interface{})

blobStorageLocation := datafactory.AzureBlobFSLocation{
FileSystem: props["file_system"].(string),
blobFSLocation := datafactory.AzureBlobFSLocation{
Type: datafactory.TypeBasicDatasetLocationTypeAzureBlobFSLocation,
}
if path := props["path"].(string); len(path) > 0 {
blobStorageLocation.FolderPath = path
}
if filename := props["filename"].(string); len(filename) > 0 {
blobStorageLocation.FileName = filename
FileSystem: expandDataFactoryExpressionResultType(props["file_system"].(string), props["dynamic_file_system_enabled"].(bool)),
FolderPath: expandDataFactoryExpressionResultType(props["path"].(string), props["dynamic_path_enabled"].(bool)),
FileName: expandDataFactoryExpressionResultType(props["filename"].(string), props["dynamic_filename_enabled"].(bool)),
}

return blobStorageLocation
return blobFSLocation
}

func flattenDataFactoryDatasetHTTPServerLocation(input *datafactory.HTTPServerLocation) []interface{} {
Expand Down Expand Up @@ -490,31 +486,25 @@ func flattenDataFactoryDatasetAzureBlobFSLocation(input *datafactory.AzureBlobFS
if input == nil {
return []interface{}{}
}
result := make(map[string]interface{})

fileSystem, path, fileName := "", "", ""
if input.FileSystem != nil {
if v, ok := input.FileSystem.(string); ok {
fileSystem = v
}
fileSystem, dynamicFileSystemEnabled := flattenDataFactoryExpressionResultType(input.FileSystem)
result["file_system"] = fileSystem
result["dynamic_file_system_enabled"] = dynamicFileSystemEnabled
}
if input.FolderPath != nil {
if v, ok := input.FolderPath.(string); ok {
path = v
}
path, dynamicPathEnabled := flattenDataFactoryExpressionResultType(input.FolderPath)
result["path"] = path
result["dynamic_path_enabled"] = dynamicPathEnabled
}
if input.FileName != nil {
if v, ok := input.FileName.(string); ok {
fileName = v
}
filename, dynamicFilenameEnabled := flattenDataFactoryExpressionResultType(input.FileName)
result["filename"] = filename
result["dynamic_filename_enabled"] = dynamicFilenameEnabled
}

return []interface{}{
map[string]interface{}{
"file_system": fileSystem,
"path": path,
"filename": fileName,
},
}
return []interface{}{result}
}

func flattenDataFactoryDatasetSFTPLocation(input *datafactory.SftpLocation) []interface{} {
Expand Down
19 changes: 17 additions & 2 deletions internal/services/datafactory/data_factory_dataset_delimited_text_resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,19 +147,34 @@ func resourceDataFactoryDatasetDelimitedText() *pluginsdk.Resource {
Schema: map[string]*pluginsdk.Schema{
"file_system": {
Type: pluginsdk.TypeString,
Required: true,
Optional: true,
ValidateFunc: validation.StringIsNotEmpty,
},
"dynamic_file_system_enabled": {
Type: pluginsdk.TypeBool,
Optional: true,
Default: false,
},
"path": {
Type: pluginsdk.TypeString,
Optional: true,
ValidateFunc: validation.StringIsNotEmpty,
},
"dynamic_path_enabled": {
Type: pluginsdk.TypeBool,
Optional: true,
Default: false,
},
"filename": {
Type: pluginsdk.TypeString,
Optional: true,
ValidateFunc: validation.StringIsNotEmpty,
},
"dynamic_filename_enabled": {
Type: pluginsdk.TypeBool,
Optional: true,
Default: false,
},
},
},
},
Expand Down Expand Up @@ -344,7 +359,7 @@ func resourceDataFactoryDatasetDelimitedTextCreateUpdate(d *pluginsdk.ResourceDa

location := expandDataFactoryDatasetLocation(d)
if location == nil {
return fmt.Errorf("one of `http_server_location`, `azure_blob_storage_location` must be specified to create a DataFactory Delimited Text Dataset")
return fmt.Errorf("one of `http_server_location`, `azure_blob_fs_location`, `azure_blob_storage_location` must be specified to create a DataFactory Delimited Text Dataset")
}

delimited_textDatasetProperties := datafactory.DelimitedTextDatasetTypeProperties{
Expand Down
79 changes: 79 additions & 0 deletions internal/services/datafactory/data_factory_dataset_delimited_text_resource_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,13 @@ func TestAccDataFactoryDatasetDelimitedText_blobFS(t *testing.T) {
),
},
data.ImportStep(),
{
Config: r.blobFSDynamicPath(data),
Check: acceptance.ComposeTestCheckFunc(
check.That(data.ResourceName).ExistsInAzure(r),
),
},
data.ImportStep(),
})
}

Expand Down Expand Up @@ -623,6 +630,78 @@ resource "azurerm_data_factory_dataset_delimited_text" "test" {
}
`, data.RandomInteger, data.Locations.Primary, data.RandomString, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger)
}
func (DatasetDelimitedTextResource) blobFSDynamicPath(data acceptance.TestData) string {
return fmt.Sprintf(`
provider "azurerm" {
features {}
}

resource "azurerm_resource_group" "test" {
name = "acctestRG-df-%d"
location = "%s"
}

resource "azurerm_storage_account" "test" {
name = "acctestsa%s"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
account_kind = "BlobStorage"
account_tier = "Standard"
account_replication_type = "LRS"
is_hns_enabled = true
allow_nested_items_to_be_public = true
}

resource "azurerm_storage_data_lake_gen2_filesystem" "test" {
name = "acctest-datalake-%d"
storage_account_id = azurerm_storage_account.test.id
}

resource "azurerm_data_factory" "test" {
name = "acctestdf%d"
location = azurerm_resource_group.test.location
resource_group_name = azurerm_resource_group.test.name

identity {
type = "SystemAssigned"
}
}

resource "azurerm_role_assignment" "test" {
scope = azurerm_storage_account.test.id
role_definition_name = "Storage Blob Data Owner"
principal_id = azurerm_data_factory.test.identity.0.principal_id
}

resource "azurerm_data_factory_linked_service_data_lake_storage_gen2" "test" {
name = "acctestDataLakeStorage%d"
data_factory_id = azurerm_data_factory.test.id
use_managed_identity = true
url = azurerm_storage_account.test.primary_dfs_endpoint
}

resource "azurerm_data_factory_dataset_delimited_text" "test" {
name = "acctestds%d"
data_factory_id = azurerm_data_factory.test.id
linked_service_name = azurerm_data_factory_linked_service_data_lake_storage_gen2.test.name

azure_blob_fs_location {
file_system = azurerm_storage_data_lake_gen2_filesystem.test.name
dynamic_file_system_enabled = true
path = "@concat('foo/bar/',formatDateTime(convertTimeZone(utcnow(),'UTC','W. Europe Standard Time'),'yyyy-MM-dd'))"
dynamic_path_enabled = true
}

column_delimiter = ","
row_delimiter = "NEW"
encoding = "UTF-8"
quote_character = "x"
escape_character = "f"
first_row_as_header = true
null_value = "NULL"
}
`, data.RandomInteger, data.Locations.Primary, data.RandomString, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger)
}

func (DatasetDelimitedTextResource) blobDynamicContainer(data acceptance.TestData) string {
return fmt.Sprintf(`
Expand Down
65 changes: 56 additions & 9 deletions internal/services/datafactory/data_factory_dataset_parquet_resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,54 @@ func resourceDataFactoryDatasetParquet() *pluginsdk.Resource {
},
},

// Parquet Specific Field, one option for 'location'
"azure_blob_fs_location": {
Type: pluginsdk.TypeList,
MaxItems: 1,
Optional: true,
ExactlyOneOf: []string{"azure_blob_fs_location", "azure_blob_storage_location", "http_server_location"},
Elem: &pluginsdk.Resource{
Schema: map[string]*pluginsdk.Schema{
"file_system": {
Type: pluginsdk.TypeString,
Optional: true,
ValidateFunc: validation.StringIsNotEmpty,
},
"dynamic_file_system_enabled": {
Type: pluginsdk.TypeBool,
Optional: true,
Default: false,
},
"path": {
Type: pluginsdk.TypeString,
Optional: true,
ValidateFunc: validation.StringIsNotEmpty,
},
"dynamic_path_enabled": {
Type: pluginsdk.TypeBool,
Optional: true,
Default: false,
},
"filename": {
Type: pluginsdk.TypeString,
Optional: true,
ValidateFunc: validation.StringIsNotEmpty,
},
"dynamic_filename_enabled": {
Type: pluginsdk.TypeBool,
Optional: true,
Default: false,
},
},
},
},

// Parquet Specific Field, one option for 'location'
"azure_blob_storage_location": {
Type: pluginsdk.TypeList,
MaxItems: 1,
Optional: true,
ConflictsWith: []string{"http_server_location"},
Type: pluginsdk.TypeList,
MaxItems: 1,
Optional: true,
ExactlyOneOf: []string{"azure_blob_fs_location", "azure_blob_storage_location", "http_server_location"},
Elem: &pluginsdk.Resource{
Schema: map[string]*pluginsdk.Schema{
"container": {
Expand Down Expand Up @@ -156,10 +198,10 @@ func resourceDataFactoryDatasetParquet() *pluginsdk.Resource {

// Parquet Specific Field, one option for 'location'
"http_server_location": {
Type: pluginsdk.TypeList,
MaxItems: 1,
Optional: true,
ConflictsWith: []string{"azure_blob_storage_location"},
Type: pluginsdk.TypeList,
MaxItems: 1,
Optional: true,
ExactlyOneOf: []string{"azure_blob_fs_location", "azure_blob_storage_location", "http_server_location"},
Elem: &pluginsdk.Resource{
Schema: map[string]*pluginsdk.Schema{
"relative_url": {
Expand Down Expand Up @@ -270,7 +312,7 @@ func resourceDataFactoryDatasetParquetCreateUpdate(d *pluginsdk.ResourceData, me

location := expandDataFactoryDatasetLocation(d)
if location == nil {
return fmt.Errorf("One of `http_server_location`, `azure_blob_storage_location` must be specified to create a DataFactory Parquet Dataset")
return fmt.Errorf("One of `http_server_location`, `azure_blob_fs_location`, `azure_blob_storage_location` must be specified to create a DataFactory Parquet Dataset")
}

parquetDatasetProperties := datafactory.ParquetDatasetTypeProperties{
Expand Down Expand Up @@ -394,6 +436,11 @@ func resourceDataFactoryDatasetParquetRead(d *pluginsdk.ResourceData, meta inter
return fmt.Errorf("setting `azure_blob_storage_location` for Data Factory Parquet Dataset %s", err)
}
}
if azureBlobFSLocation, ok := properties.Location.AsAzureBlobFSLocation(); ok {
if err := d.Set("azure_blob_fs_location", flattenDataFactoryDatasetAzureBlobFSLocation(azureBlobFSLocation)); err != nil {
return fmt.Errorf("setting `azure_blob_fs_location` for Data Factory Parquet Dataset %s", err)
}
}

compressionCodec, ok := properties.CompressionCodec.(string)
if !ok {
Expand Down