Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Chunked file transfers for azurerm_data_lake_store_file #2633

Merged
merged 11 commits into from
Jan 11, 2019
Merged

Chunked file transfers for azurerm_data_lake_store_file #2633

Changes from 2 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
1cfaa13
implement workaround for chunked file transfers to ADLS https://githu…
Jan 9, 2019
5c77519
pass only the read n bytes of the buffer to NewReader so that the las…
Jan 10, 2019
35052a7
acceptance test for azurerm_data_lake_store_file that tests large (gr…
Jan 11, 2019
f31e8c8
renamed bufferSize to chunkSize for clarity
Jan 11, 2019
9ab2085
Merge branch 'master' into master-chunked
katbyte Jan 11, 2019
f23da35
TestAccAzureRMDataLakeStoreFile_largefiles: use a temp file with rand…
katbyte Jan 11, 2019
62d7005
fix linting issues
katbyte Jan 11, 2019
4a4f5b3
minor style fix
katbyte Jan 11, 2019
677a032
Merge branch 'master' into master-chunked
katbyte Jan 11, 2019
ef25823
acctest.RandInt() -> tf.AccRandTimeInt()
katbyte Jan 11, 2019
09fd2b9
fix import issues
katbyte Jan 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 21 additions & 6 deletions azurerm/resource_arm_data_lake_store_file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package azurerm
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"log"
"net/url"
Expand Down Expand Up @@ -53,6 +54,7 @@ func resourceArmDataLakeStoreFile() *schema.Resource {
func resourceArmDataLakeStoreFileCreate(d *schema.ResourceData, meta interface{}) error {
client := meta.(*ArmClient).dataLakeStoreFilesClient
ctx := meta.(*ArmClient).StopContext
bufferSize := 4 * 1024 * 1024

log.Printf("[INFO] preparing arguments for Date Lake Store File creation.")

Expand Down Expand Up @@ -82,15 +84,28 @@ func resourceArmDataLakeStoreFileCreate(d *schema.ResourceData, meta interface{}
}
defer utils.IoCloseAndLogError(file, fmt.Sprintf("Error closing Data Lake Store File %q", localFilePath))

// Read the file contents
fileContents, err := ioutil.ReadAll(file)
_, err = client.Create(ctx, accountName, remoteFilePath, nil, nil, filesystem.DATA, nil, nil)
if err != nil {
return err
return fmt.Errorf("Error issuing create request for Data Lake Store File %q : %+v", remoteFilePath, err)
}

_, err = client.Create(ctx, accountName, remoteFilePath, ioutil.NopCloser(bytes.NewReader(fileContents)), utils.Bool(false), filesystem.CLOSE, nil, nil)
if err != nil {
return fmt.Errorf("Error issuing create request for Data Lake Store File %q : %+v", remoteFilePath, err)
buffer := make([]byte, bufferSize, bufferSize)
for {
n, err := file.Read(buffer)
if err == io.EOF {
break
}
flag := filesystem.DATA
if n < bufferSize {
// last chunk
flag = filesystem.CLOSE
}
chunk := ioutil.NopCloser(bytes.NewReader(buffer[:n]))

_, err = client.Append(ctx, accountName, remoteFilePath, chunk, nil, flag, nil, nil)
if err != nil {
return fmt.Errorf("Error transferring chunk for Data Lake Store File %q : %+v", remoteFilePath, err)
}
}

d.SetId(id)
Expand Down