diff --git a/cmd/zt_interceptors_for_test.go b/cmd/zt_interceptors_for_test.go index 9bdbcfeb9..6fad151c9 100644 --- a/cmd/zt_interceptors_for_test.go +++ b/cmd/zt_interceptors_for_test.go @@ -82,6 +82,10 @@ type mockedLifecycleManager struct { outputFormat common.OutputFormat } +func (m *mockedLifecycleManager) DownloadToTempPath() bool { + return false +} + func (m *mockedLifecycleManager) Progress(o common.OutputBuilder) { select { case m.progressLog <- o(common.EOutputFormat.Text()): diff --git a/common/environment.go b/common/environment.go index b0cae578a..203160ca4 100644 --- a/common/environment.go +++ b/common/environment.go @@ -361,3 +361,11 @@ func (EnvironmentVariable) MimeMapping() EnvironmentVariable { Description: "Location of the file to override default OS mime mapping", } } + +func (EnvironmentVariable) DownloadToTempPath() EnvironmentVariable { + return EnvironmentVariable { + Name: "AZCOPY_DOWNLOAD_TO_TEMP_PATH", + DefaultValue: "true", + Description: "Configures azcopy to download to a temp path before actual download. Allowed values are true/false", + } +} diff --git a/common/lifecyleMgr.go b/common/lifecyleMgr.go index 1b6107ad8..2d21417d7 100644 --- a/common/lifecyleMgr.go +++ b/common/lifecyleMgr.go @@ -68,6 +68,7 @@ type LifecycleMgr interface { RegisterCloseFunc(func()) SetForceLogging() IsForceLoggingDisabled() bool + DownloadToTempPath() bool } func GetLifecycleMgr() LifecycleMgr { @@ -590,6 +591,15 @@ func (lcm *lifecycleMgr) IsForceLoggingDisabled() bool { return lcm.disableSyslog } +func (lcm *lifecycleMgr) DownloadToTempPath() bool { + ret, err := strconv.ParseBool(lcm.GetEnvironmentVariable(EEnvironmentVariable.DownloadToTempPath())) + if err != nil { + // By default we'll download to temp path + ret = true + } + return ret +} + // captures the common logic of exiting if there's an expected error func PanicIfErr(err error) { if err != nil { diff --git a/ste/xfer-remoteToLocal-file.go b/ste/xfer-remoteToLocal-file.go index b09f55a38..5b93c8a26 100644 --- a/ste/xfer-remoteToLocal-file.go +++ b/ste/xfer-remoteToLocal-file.go @@ -171,7 +171,7 @@ func remoteToLocal_file(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer pac // to correct name. pseudoId := common.NewPseudoChunkIDForWholeFile(info.Source) jptm.LogChunkStatus(pseudoId, common.EWaitReason.CreateLocalFile()) - dstFile, err = createDestinationFile(jptm, info.getTempDownloadPath(), fileSize, writeThrough) + dstFile, err = createDestinationFile(jptm, info.getDownloadPath(), fileSize, writeThrough) jptm.LogChunkStatus(pseudoId, common.EWaitReason.ChunkDone()) // normal setting to done doesn't apply to these pseudo ids if err != nil { failFileCreation(err) @@ -342,7 +342,7 @@ func epilogueWithCleanupDownload(jptm IJobPartTransferMgr, dl downloader, active // check length if enabled (except for dev null and decompression case, where that's impossible) if info.DestLengthValidation && info.Destination != common.Dev_Null && !jptm.ShouldDecompress() { - fi, err := common.OSStat(info.getTempDownloadPath()) + fi, err := common.OSStat(info.getDownloadPath()) if err != nil { jptm.FailActiveDownload("Download length check", err) @@ -351,14 +351,16 @@ func epilogueWithCleanupDownload(jptm IJobPartTransferMgr, dl downloader, active } } - //Rename back to original name. At this point, we're sure the file is completely + //check if we need to rename back to original name. At this point, we're sure the file is completely //downloaded and not corrupt. Infact, post this point we should only log errors and //not fail the transfer. - if err == nil && !strings.EqualFold(info.Destination, common.Dev_Null) { - renameErr := os.Rename(info.getTempDownloadPath(), info.Destination) + renameNecessary := !strings.EqualFold(info.getDownloadPath(), info.Destination) && + !strings.EqualFold(info.Destination, common.Dev_Null) + if err == nil && renameNecessary { + renameErr := os.Rename(info.getDownloadPath(), info.Destination) if renameErr != nil { jptm.LogError(info.Destination, fmt.Sprintf( - "Failed to rename. File at %s", info.getTempDownloadPath()), renameErr) + "Failed to rename. File at %s", info.getDownloadPath()), renameErr) } } } @@ -461,19 +463,23 @@ func tryDeleteFile(info TransferInfo, jptm IJobPartTransferMgr) { return } - err := deleteFile(info.getTempDownloadPath()) + err := deleteFile(info.getDownloadPath()) if err != nil { // If there was an error deleting the file, log the error jptm.LogError(info.Destination, "Delete File Error ", err) } } -//Returns the path of temp file to be downloaded. The paths is in format +// Returns the path of file to be downloaded. If we want to +// download to a temp path we return a temp paht in format // /actual/parent/path/.azDownload-- -func (info *TransferInfo) getTempDownloadPath() string { - parent, fileName := filepath.Split(info.Destination) - fileName = fmt.Sprintf(azcopyTempDownloadPrefix, info.JobID.String()) + fileName - return filepath.Join(parent, fileName) +func (info *TransferInfo) getDownloadPath() string { + if common.GetLifecycleMgr().DownloadToTempPath() { + parent, fileName := filepath.Split(info.Destination) + fileName = fmt.Sprintf(azcopyTempDownloadPrefix, info.JobID.String()) + fileName + return filepath.Join(parent, fileName) + } + return info.Destination } // conforms to io.Writer and io.Closer