From 62b3d47b43d91109586b352c22844146c495c4dc Mon Sep 17 00:00:00 2001 From: Andrey Prygunkov Date: Fri, 28 Apr 2017 23:55:41 +0200 Subject: [PATCH] #362: redesigned renaming Spread RenameInfo into FileInfo and NzbInfo to make the state easier to save on disk --- daemon/nntp/ArticleWriter.cpp | 6 +- daemon/postprocess/PrePostProcessor.cpp | 4 +- daemon/postprocess/Rename.cpp | 6 +- daemon/postprocess/Rename.h | 6 +- daemon/postprocess/Repair.cpp | 4 +- daemon/queue/DirectRenamer.cpp | 289 +++++++++++--------- daemon/queue/DirectRenamer.h | 40 ++- daemon/queue/DiskState.cpp | 9 +- daemon/queue/DownloadInfo.cpp | 17 +- daemon/queue/DownloadInfo.h | 116 ++++---- daemon/queue/HistoryCoordinator.cpp | 18 +- daemon/queue/QueueCoordinator.cpp | 26 +- tests/functional/rename/rename_opt2_test.py | 30 +- 13 files changed, 322 insertions(+), 249 deletions(-) diff --git a/daemon/nntp/ArticleWriter.cpp b/daemon/nntp/ArticleWriter.cpp index a2917c92f..35fc58011 100644 --- a/daemon/nntp/ArticleWriter.cpp +++ b/daemon/nntp/ArticleWriter.cpp @@ -690,14 +690,14 @@ bool ArticleWriter::MoveCompletedFiles(NzbInfo* nzbInfo, const char* oldDestDir) // move already downloaded files to new destination for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles()) { - BString<1024> oldFileName("%s%c%s", oldDestDir, (int)PATH_SEPARATOR, completedFile.GetFileName()); - BString<1024> newFileName("%s%c%s", nzbInfo->GetDestDir(), (int)PATH_SEPARATOR, completedFile.GetFileName()); + BString<1024> oldFileName("%s%c%s", oldDestDir, (int)PATH_SEPARATOR, completedFile.GetFilename()); + BString<1024> newFileName("%s%c%s", nzbInfo->GetDestDir(), (int)PATH_SEPARATOR, completedFile.GetFilename()); // check if file was not moved already if (strcmp(oldFileName, newFileName)) { // prevent overwriting of existing files - newFileName = FileSystem::MakeUniqueFilename(nzbInfo->GetDestDir(), completedFile.GetFileName()); + newFileName = FileSystem::MakeUniqueFilename(nzbInfo->GetDestDir(), completedFile.GetFilename()); detail("Moving file %s to %s", *oldFileName, *newFileName); if (!FileSystem::MoveFile(oldFileName, newFileName)) diff --git a/daemon/postprocess/PrePostProcessor.cpp b/daemon/postprocess/PrePostProcessor.cpp index 1a0a0f3e6..d33172588 100644 --- a/daemon/postprocess/PrePostProcessor.cpp +++ b/daemon/postprocess/PrePostProcessor.cpp @@ -359,10 +359,10 @@ void PrePostProcessor::DeleteCleanup(NzbInfo* nzbInfo) // download was cancelled, deleting already downloaded files from disk for (CompletedFile& completedFile: nzbInfo->GetCompletedFiles()) { - BString<1024> fullFileName("%s%c%s", nzbInfo->GetDestDir(), (int)PATH_SEPARATOR, completedFile.GetFileName()); + BString<1024> fullFileName("%s%c%s", nzbInfo->GetDestDir(), (int)PATH_SEPARATOR, completedFile.GetFilename()); if (FileSystem::FileExists(fullFileName)) { - detail("Deleting file %s", completedFile.GetFileName()); + detail("Deleting file %s", completedFile.GetFilename()); FileSystem::DeleteFile(fullFileName); } } diff --git a/daemon/postprocess/Rename.cpp b/daemon/postprocess/Rename.cpp index 74282496b..9e6f157dd 100644 --- a/daemon/postprocess/Rename.cpp +++ b/daemon/postprocess/Rename.cpp @@ -207,13 +207,13 @@ void RenameController::UpdateRarRenameProgress() /** * Update file name in the CompletedFiles-list of NZBInfo */ -void RenameController::RegisterRenamedFile(const char* oldFilename, const char* newFileName) +void RenameController::RegisterRenamedFile(const char* oldFilename, const char* newFilename) { for (CompletedFile& completedFile : m_postInfo->GetNzbInfo()->GetCompletedFiles()) { - if (!strcasecmp(completedFile.GetFileName(), oldFilename)) + if (!strcasecmp(completedFile.GetFilename(), oldFilename)) { - completedFile.SetFileName(newFileName); + completedFile.SetFilename(newFilename); break; } } diff --git a/daemon/postprocess/Rename.h b/daemon/postprocess/Rename.h index 4ebb9a9b1..32ff7a379 100644 --- a/daemon/postprocess/Rename.h +++ b/daemon/postprocess/Rename.h @@ -78,8 +78,8 @@ class RenameController : public Thread, public ScriptController protected: virtual void UpdateProgress() { m_owner->UpdateRarRenameProgress(); } virtual void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3); - virtual void RegisterRenamedFile(const char* oldFilename, const char* newFileName) - { m_owner->RegisterRenamedFile(oldFilename, newFileName); } + virtual void RegisterRenamedFile(const char* oldFilename, const char* newFilename) + { m_owner->RegisterRenamedFile(oldFilename, newFilename); } virtual bool IsStopped() { return m_owner->IsStopped(); }; private: RenameController* m_owner; @@ -92,7 +92,7 @@ class RenameController : public Thread, public ScriptController void ExecRename(const char* destDir, const char* finalDir, const char* nzbName); void RenameCompleted(); - void RegisterRenamedFile(const char* oldFilename, const char* newFileName); + void RegisterRenamedFile(const char* oldFilename, const char* newFilename); }; #endif diff --git a/daemon/postprocess/Repair.cpp b/daemon/postprocess/Repair.cpp index 38e2547da..d6731d6af 100644 --- a/daemon/postprocess/Repair.cpp +++ b/daemon/postprocess/Repair.cpp @@ -74,7 +74,7 @@ ParChecker::EFileStatus RepairController::PostParChecker::FindFileCrc(const char for (CompletedFile& completedFile2 : m_postInfo->GetNzbInfo()->GetCompletedFiles()) { - if (!strcasecmp(completedFile2.GetFileName(), filename)) + if (!strcasecmp(completedFile2.GetFilename(), filename)) { completedFile = &completedFile2; break; @@ -85,7 +85,7 @@ ParChecker::EFileStatus RepairController::PostParChecker::FindFileCrc(const char return ParChecker::fsUnknown; } - debug("Found completed file: %s, CRC: %.8x, Status: %i", FileSystem::BaseFileName(completedFile->GetFileName()), completedFile->GetCrc(), (int)completedFile->GetStatus()); + debug("Found completed file: %s, CRC: %.8x, Status: %i", FileSystem::BaseFileName(completedFile->GetFilename()), completedFile->GetCrc(), (int)completedFile->GetStatus()); *crc = completedFile->GetCrc(); diff --git a/daemon/queue/DirectRenamer.cpp b/daemon/queue/DirectRenamer.cpp index 048d231be..fabe55b2b 100644 --- a/daemon/queue/DirectRenamer.cpp +++ b/daemon/queue/DirectRenamer.cpp @@ -71,7 +71,7 @@ class DirectParLoader : public Thread private: DirectRenamer* m_owner; NameList m_parFiles; - RenameInfo::FileHashList m_parHashes; + DirectRenamer::FileHashList m_parHashes; int m_nzbId; void LoadParFile(const char* parFile); @@ -84,12 +84,12 @@ void DirectParLoader::StartLoader(DirectRenamer* owner, NzbInfo* nzbInfo) directParLoader->m_owner = owner; directParLoader->m_nzbId = nzbInfo->GetId(); - for (RenameInfo::ParFile& parFile : nzbInfo->GetRenameInfo()->GetParFiles()) + for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles()) { - if (parFile.GetCompleted()) + if (completedFile.GetParFile()) { directParLoader->m_parFiles.emplace_back(BString<1024>("%s%c%s", - nzbInfo->GetDestDir(), PATH_SEPARATOR, parFile.GetFilename())); + nzbInfo->GetDestDir(), PATH_SEPARATOR, completedFile.GetFilename())); } } @@ -175,11 +175,10 @@ void DirectRenamer::ArticleDownloaded(DownloadQueue* downloadQueue, FileInfo* fi NzbInfo* nzbInfo = fileInfo->GetNzbInfo(); // we don't support analyzing of files split into articles smaller than 16KB - if (!contentAnalyzer->GetParFile() && - (articleInfo->GetSize() >= 16 * 1024 || fileInfo->GetArticles()->size() == 1)) + if (articleInfo->GetSize() >= 16 * 1024 || fileInfo->GetArticles()->size() == 1) { - nzbInfo->GetRenameInfo()->GetArticleHashes()->emplace_back(fileInfo->GetFilename(), contentAnalyzer->GetHash16k()); - debug("file: %s; article-hash16k: %s", fileInfo->GetFilename(), contentAnalyzer->GetHash16k()); + fileInfo->SetHash16k(contentAnalyzer->GetHash16k()); + debug("file: %s; article-hash16k: %s", fileInfo->GetFilename(), fileInfo->GetHash16k()); } detail("Detected %s %s", (contentAnalyzer->GetParFile() ? "par2-file" : "non-par2-file"), fileInfo->GetFilename()); @@ -202,9 +201,8 @@ void DirectRenamer::ArticleDownloaded(DownloadQueue* downloadQueue, FileInfo* fi if (fileInfo->GetParFile()) { - nzbInfo->GetRenameInfo()->GetParFiles()->emplace_back(fileInfo->GetId(), - fileInfo->GetFilename(), contentAnalyzer->GetParSetId()); - debug("file: %s; setid: %s", fileInfo->GetFilename(), contentAnalyzer->GetParSetId()); + fileInfo->SetParSetId(contentAnalyzer->GetParSetId()); + debug("file: %s; setid: %s", fileInfo->GetFilename(), fileInfo->GetParSetId()); } CheckState(nzbInfo); @@ -212,51 +210,51 @@ void DirectRenamer::ArticleDownloaded(DownloadQueue* downloadQueue, FileInfo* fi void DirectRenamer::FileDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo) { - if (fileInfo->GetParFile()) - { - RenameInfo::ParFileList::iterator pos = std::find_if( - fileInfo->GetNzbInfo()->GetRenameInfo()->GetParFiles()->begin(), - fileInfo->GetNzbInfo()->GetRenameInfo()->GetParFiles()->end(), - [id = fileInfo->GetId()](RenameInfo::ParFile& parFile) - { - return parFile.GetId() == id; - }); - if (pos != fileInfo->GetNzbInfo()->GetRenameInfo()->GetParFiles()->end()) - { - (*pos).SetCompleted(true); - } - } - CheckState(fileInfo->GetNzbInfo()); } void DirectRenamer::CheckState(NzbInfo* nzbInfo) { - if (nzbInfo->GetRenameInfo()->GetArticleHashes()->size() + - nzbInfo->GetRenameInfo()->GetParFiles()->size() == nzbInfo->GetFileCount() && - !nzbInfo->GetRenameInfo()->GetWaitingPar()) + if (nzbInfo->GetDirectRenameStatus() > NzbInfo::tsRunning) + { + return; + } + + // check if all first articles are downloaded + FileList::iterator pos = std::find_if( + nzbInfo->GetFileList()->begin(), nzbInfo->GetFileList()->end(), + [](std::unique_ptr& fileInfo) + { + return Util::EmptyStr(fileInfo->GetHash16k()); + }); + + if (pos != nzbInfo->GetFileList()->end()) + { + return; + } + + if (!nzbInfo->GetWaitingPar()) { // all first articles downloaded UnpausePars(nzbInfo); - nzbInfo->GetRenameInfo()->SetWaitingPar(true); + nzbInfo->SetWaitingPar(true); } - if (nzbInfo->GetRenameInfo()->GetWaitingPar() && - !nzbInfo->GetRenameInfo()->GetLoadingPar()) + if (nzbInfo->GetWaitingPar() && !nzbInfo->GetLoadingPar()) { // check if all par2-files scheduled for downloading already completed - RenameInfo::ParFileList::iterator pos = std::find_if( - nzbInfo->GetRenameInfo()->GetParFiles()->begin(), - nzbInfo->GetRenameInfo()->GetParFiles()->end(), - [](RenameInfo::ParFile& parFile) - { - return parFile.GetWanted() && !parFile.GetCompleted(); - }); - if (pos == nzbInfo->GetRenameInfo()->GetParFiles()->end()) + FileList::iterator pos = std::find_if( + nzbInfo->GetFileList()->begin(), nzbInfo->GetFileList()->end(), + [](std::unique_ptr& fileInfo) + { + return fileInfo->GetExtraPriority(); + }); + + if (pos == nzbInfo->GetFileList()->end()) { // all wanted par2-files are downloaded detail("Loading par2-files for direct renaming"); - nzbInfo->GetRenameInfo()->SetLoadingPar(true); + nzbInfo->SetLoadingPar(true); DirectParLoader::StartLoader(this, nzbInfo); return; } @@ -266,32 +264,32 @@ void DirectRenamer::CheckState(NzbInfo* nzbInfo) // Unpause smallest par-files from each par-set void DirectRenamer::UnpausePars(NzbInfo* nzbInfo) { + ParFileList parFiles; + CollectPars(nzbInfo, &parFiles); + std::vector parsets; // sort by size - std::sort( - nzbInfo->GetRenameInfo()->GetParFiles()->begin(), - nzbInfo->GetRenameInfo()->GetParFiles()->end(), - [nzbInfo](const RenameInfo::ParFile& parFile1, const RenameInfo::ParFile& parFile2) + std::sort(parFiles.begin(), parFiles.end(), + [nzbInfo](const ParFile& parFile1, const ParFile& parFile2) { - FileInfo* fileInfo1 = nzbInfo->GetFileList()->Find(const_cast(parFile1).GetId()); - FileInfo* fileInfo2 = nzbInfo->GetFileList()->Find(const_cast(parFile2).GetId()); + FileInfo* fileInfo1 = nzbInfo->GetFileList()->Find(const_cast(parFile1).GetId()); + FileInfo* fileInfo2 = nzbInfo->GetFileList()->Find(const_cast(parFile2).GetId()); return (!fileInfo1 && fileInfo2) || (fileInfo1 && fileInfo2 && fileInfo1->GetSize() < fileInfo2->GetSize()); }); // 1. count already downloaded files - for (RenameInfo::ParFile& parFile : nzbInfo->GetRenameInfo()->GetParFiles()) + for (ParFile& parFile : parFiles) { if (parFile.GetCompleted()) { parsets.emplace_back(parFile.GetSetId()); - parFile.SetWanted(true); } } // 2. find smallest par-file from each par-set from not yet completely downloaded files - for (RenameInfo::ParFile& parFile : nzbInfo->GetRenameInfo()->GetParFiles()) + for (ParFile& parFile : parFiles) { std::vector::iterator pos = std::find(parsets.begin(), parsets.end(), parFile.GetSetId()); if (pos == parsets.end()) @@ -305,134 +303,163 @@ void DirectRenamer::UnpausePars(NzbInfo* nzbInfo) nzbInfo->PrintMessage(Message::mkDetail, "Increasing priority for par2-file %s", fileInfo->GetFilename()); fileInfo->SetPaused(false); fileInfo->SetExtraPriority(true); - parFile.SetWanted(true); } } } } -void DirectRenamer::RenameFiles(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, RenameInfo::FileHashList* parHashes) +void DirectRenamer::CollectPars(NzbInfo* nzbInfo, ParFileList* parFiles) { - // rename regular files - for (RenameInfo::FileHash& parHash : parHashes) + for (FileInfo* fileInfo : nzbInfo->GetFileList()) { - RenameInfo::FileHashList::iterator pos = std::find_if( - nzbInfo->GetRenameInfo()->GetArticleHashes()->begin(), - nzbInfo->GetRenameInfo()->GetArticleHashes()->end(), - [&parHash](RenameInfo::FileHash& articleHash) - { - return !strcmp(parHash.GetHash(), articleHash.GetHash()); - }); - if (pos != nzbInfo->GetRenameInfo()->GetArticleHashes()->end()) + if (fileInfo->GetParFile()) { - RenameInfo::FileHash& articleHash = *pos; - if (strcasecmp(articleHash.GetFilename(), parHash.GetFilename())) - { - RenameFile(nzbInfo, articleHash.GetFilename(), parHash.GetFilename()); - } - nzbInfo->GetRenameInfo()->GetArticleHashes()->erase(pos); + parFiles->emplace_back(fileInfo->GetId(), fileInfo->GetFilename(), fileInfo->GetParSetId(), false); + } + } + + for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles()) + { + if (completedFile.GetParFile()) + { + parFiles->emplace_back(completedFile.GetId(), completedFile.GetFilename(), completedFile.GetParSetId(), true); } } +} + +void DirectRenamer::RenameFiles(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, FileHashList* parHashes) +{ + bool renamePars = NeedRenamePars(nzbInfo); + int vol = 1; - // rename par2-files - if (NeedRenamePars(nzbInfo)) + // rename in-progress files + for (FileInfo* fileInfo : nzbInfo->GetFileList()) { - int num = 1; - for (RenameInfo::ParFile& parFile : nzbInfo->GetRenameInfo()->GetParFiles()) + CString newName; + if (fileInfo->GetParFile() && renamePars) { - BString<1024> newName; - BString<1024> destFileName; + newName = BuildNewParName(fileInfo->GetFilename(), nzbInfo->GetDestDir(), fileInfo->GetParSetId(), vol); + } + else if (!fileInfo->GetParFile()) + { + newName = BuildNewRegularName(fileInfo->GetFilename(), parHashes, fileInfo->GetHash16k()); + } - // trying to reuse file suffix - const char* suffix = strstr(parFile.GetFilename(), ".vol"); - const char* extension = suffix ? strrchr(suffix, '.') : nullptr; - if (suffix && extension && !strcasecmp(extension, ".par2")) + if (newName) + { + bool written = fileInfo->GetOutputFilename() && + !Util::EndsWith(fileInfo->GetOutputFilename(), ".out.tmp"); + if (!written) { - newName.Format("%s%s", parFile.GetSetId(), suffix); - destFileName.Format("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, *newName); + nzbInfo->PrintMessage(Message::mkInfo, "Renaming in-progress file %s to %s", + fileInfo->GetFilename(), *newName); + fileInfo->SetFilename(newName); } - - while (destFileName.Empty() || FileSystem::FileExists(destFileName)) + else if (RenameCompletedFile(nzbInfo, fileInfo->GetFilename(), newName)) { - newName.Format("%s.vol%03i+01.PAR2", parFile.GetSetId(), num); - destFileName.Format("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, *newName); - num++; + fileInfo->SetFilename(newName); } - RenameFile(nzbInfo, parFile.GetFilename(), newName); + } + } + + // rename completed files + for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles()) + { + CString newName; + if (completedFile.GetParFile() && renamePars) + { + newName = BuildNewParName(completedFile.GetFilename(), nzbInfo->GetDestDir(), completedFile.GetParSetId(), vol); + } + else if (!completedFile.GetParFile()) + { + newName = BuildNewRegularName(completedFile.GetFilename(), parHashes, completedFile.GetHash16k()); + } + + if (newName && RenameCompletedFile(nzbInfo, completedFile.GetFilename(), newName)) + { + completedFile.SetFilename(newName); } } RenameCompleted(downloadQueue, nzbInfo); } -bool DirectRenamer::NeedRenamePars(NzbInfo* nzbInfo) +CString DirectRenamer::BuildNewRegularName(const char* oldName, FileHashList* parHashes, const char* hash16k) { - // renaming is needed if par2-files from same par-set have different base names + if (Util::EmptyStr(hash16k)) + { + return nullptr; + } - for (RenameInfo::ParFile& parFile : nzbInfo->GetRenameInfo()->GetParFiles()) + FileHashList::iterator pos = std::find_if(parHashes->begin(), parHashes->end(), + [hash16k](FileHash& parHash) { - int baseLen; - ParParser::ParseParFilename(parFile.GetFilename(), false, &baseLen, nullptr); - BString<1024> basename; - basename.Set(parFile.GetFilename(), baseLen); + return !strcmp(parHash.GetHash(), hash16k); + }); - for (RenameInfo::ParFile& parFile2 : nzbInfo->GetRenameInfo()->GetParFiles()) + if (pos != parHashes->end()) + { + FileHash& parHash = *pos; + if (strcasecmp(oldName, parHash.GetFilename())) { - ParParser::ParseParFilename(parFile.GetFilename(), false, &baseLen, nullptr); - BString<1024> basename2; - basename2.Set(parFile2.GetFilename(), baseLen); - - if (&parFile != &parFile2 && strcmp(basename, basename2)) - { - return true; - } + return parHash.GetFilename(); } } - return false; + return nullptr; } -void DirectRenamer::RenameFile(NzbInfo* nzbInfo, const char* oldName, const char* newName) +CString DirectRenamer::BuildNewParName(const char* oldName, const char* destDir, const char* setId, int& vol) { - BString<1024> newFullFilename("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, newName); - if (FileSystem::FileExists(newFullFilename)) + BString<1024> newName; + BString<1024> destFileName; + + // trying to reuse file suffix + const char* suffix = strstr(oldName, ".vol"); + const char* extension = suffix ? strrchr(suffix, '.') : nullptr; + if (suffix && extension && !strcasecmp(extension, ".par2")) { - nzbInfo->PrintMessage(Message::mkWarning, - "Could not rename file %s to %s: destination file already exists", - oldName, newName); - return; + newName.Format("%s%s", setId, suffix); + destFileName.Format("%s%c%s", destDir, PATH_SEPARATOR, *newName); } - for (FileInfo* fileInfo : nzbInfo->GetFileList()) + while (destFileName.Empty() || FileSystem::FileExists(destFileName)) { - if (!strcmp(fileInfo->GetFilename(), oldName)) - { - bool written = fileInfo->GetOutputFilename() && - !Util::EndsWith(fileInfo->GetOutputFilename(), ".out.tmp"); - if (written) - { - RenameCompletedFile(nzbInfo, oldName, newName); - } - else - { - nzbInfo->PrintMessage(Message::mkInfo, "Renaming in-progress file %s to %s", oldName, newName); - } - fileInfo->SetFilename(newName); - return; - } + newName.Format("%s.vol%03i+01.PAR2", setId, vol); + destFileName.Format("%s%c%s", destDir, PATH_SEPARATOR, *newName); + vol++; } - for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles()) + return *newName; +} + +bool DirectRenamer::NeedRenamePars(NzbInfo* nzbInfo) +{ + // renaming is needed if par2-files from same par-set have different base names + ParFileList parFiles; + CollectPars(nzbInfo, &parFiles); + + for (ParFile& parFile : parFiles) { - if (!strcmp(completedFile.GetFileName(), oldName)) + int baseLen; + ParParser::ParseParFilename(parFile.GetFilename(), false, &baseLen, nullptr); + BString<1024> basename; + basename.Set(parFile.GetFilename(), baseLen); + + for (ParFile& parFile2 : parFiles) { - if (RenameCompletedFile(nzbInfo, oldName, newName)) + ParParser::ParseParFilename(parFile.GetFilename(), false, &baseLen, nullptr); + BString<1024> basename2; + basename2.Set(parFile2.GetFilename(), baseLen); + + if (&parFile != &parFile2 && strcmp(basename, basename2)) { - completedFile.SetFileName(newName); + return true; } - return; } } + + return false; } bool DirectRenamer::RenameCompletedFile(NzbInfo* nzbInfo, const char* oldName, const char* newName) diff --git a/daemon/queue/DirectRenamer.h b/daemon/queue/DirectRenamer.h index 57163b5cd..9a62da887 100644 --- a/daemon/queue/DirectRenamer.h +++ b/daemon/queue/DirectRenamer.h @@ -26,6 +26,40 @@ class DirectRenamer { public: + class FileHash + { + public: + FileHash(const char* filename, const char* hash) : + m_filename(filename), m_hash(hash) {} + const char* GetFilename() { return m_filename; } + const char* GetHash() { return m_hash; } + + private: + CString m_filename; + CString m_hash; + }; + + typedef std::deque FileHashList; + + class ParFile + { + public: + ParFile(int id, const char* filename, const char* setId, bool completed) : + m_id(id), m_filename(filename), m_setId(setId), m_completed(completed) {} + int GetId() { return m_id; } + const char* GetFilename() { return m_filename; } + const char* GetSetId() { return m_setId; } + bool GetCompleted() { return m_completed; } + + private: + int m_id; + CString m_filename; + CString m_setId; + bool m_completed = false; + }; + + typedef std::deque ParFileList; + std::unique_ptr MakeArticleContentAnalyzer(); void ArticleDownloaded(DownloadQueue* downloadQueue, FileInfo* fileInfo, ArticleInfo* articleInfo, ArticleContentAnalyzer* articleContentAnalyzer); @@ -37,10 +71,12 @@ class DirectRenamer private: void CheckState(NzbInfo* nzbInfo); void UnpausePars(NzbInfo* nzbInfo); - void RenameFiles(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, RenameInfo::FileHashList* parHashes); - void RenameFile(NzbInfo* nzbInfo, const char* oldName, const char* newName); + void RenameFiles(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, FileHashList* parHashes); bool RenameCompletedFile(NzbInfo* nzbInfo, const char* oldName, const char* newName); bool NeedRenamePars(NzbInfo* nzbInfo); + void CollectPars(NzbInfo* nzbInfo, ParFileList* parFiles); + CString BuildNewRegularName(const char* oldName, FileHashList* parHashes, const char* hash16k); + CString BuildNewParName(const char* oldName, const char* destDir, const char* setId, int& vol); friend class DirectParLoader; }; diff --git a/daemon/queue/DiskState.cpp b/daemon/queue/DiskState.cpp index 672f63dc1..fc2edda2a 100644 --- a/daemon/queue/DiskState.cpp +++ b/daemon/queue/DiskState.cpp @@ -467,7 +467,7 @@ void DiskState::SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile) for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles()) { outfile.PrintLine("%i,%i,%u,%s", completedFile.GetId(), (int)completedFile.GetStatus(), - completedFile.GetCrc(), completedFile.GetFileName()); + completedFile.GetCrc(), completedFile.GetFilename()); } outfile.PrintLine("%i", (int)nzbInfo->GetParameters()->size()); @@ -577,8 +577,8 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i nzbInfo->SetParStatus((NzbInfo::EParStatus)parStatus); nzbInfo->SetUnpackStatus((NzbInfo::EUnpackStatus)unpackStatus); nzbInfo->SetMoveStatus((NzbInfo::EMoveStatus)moveStatus); - nzbInfo->SetParRenameStatus((NzbInfo::ERenameStatus)parRenameStatus); - nzbInfo->SetRarRenameStatus((NzbInfo::ERenameStatus)rarRenameStatus); + nzbInfo->SetParRenameStatus((NzbInfo::EPostRenameStatus)parRenameStatus); + nzbInfo->SetRarRenameStatus((NzbInfo::EPostRenameStatus)rarRenameStatus); nzbInfo->SetDeleteStatus((NzbInfo::EDeleteStatus)deleteStatus); nzbInfo->SetMarkStatus((NzbInfo::EMarkStatus)markStatus); if (nzbInfo->GetKind() == NzbInfo::nkNzb || @@ -711,7 +711,8 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i } } - nzbInfo->GetCompletedFiles()->emplace_back(id, fileName, (CompletedFile::EStatus)status, crc); + nzbInfo->GetCompletedFiles()->emplace_back(id, fileName, + (CompletedFile::EStatus)status, crc, false, nullptr, nullptr); } int parameterCount; diff --git a/daemon/queue/DownloadInfo.cpp b/daemon/queue/DownloadInfo.cpp index 8d391bbce..9c6b2de65 100644 --- a/daemon/queue/DownloadInfo.cpp +++ b/daemon/queue/DownloadInfo.cpp @@ -791,9 +791,10 @@ void FileInfo::SetActiveDownloads(int activeDownloads) } -CompletedFile::CompletedFile(int id, const char* fileName, EStatus status, uint32 crc) : - m_id(id), m_fileName(fileName), m_status(status), m_crc(crc) - +CompletedFile::CompletedFile(int id, const char* filename, EStatus status, uint32 crc, + bool parFile, const char* hash16k, const char* parSetId) : + m_id(id), m_filename(filename), m_status(status), m_crc(crc), m_parFile(parFile), + m_hash16k(hash16k), m_parSetId(parSetId) { if (FileInfo::m_idMax < m_id) { @@ -880,13 +881,3 @@ void DownloadQueue::CalcRemainingSize(int64* remaining, int64* remainingForced) *remainingForced = remainingForcedSize; } } - - -void RenameInfo::Reset() -{ - m_allFirst = false; - m_waitingPar = false; - m_loadingPar = false; - m_articleHashes.clear(); - m_parFiles.clear(); -} diff --git a/daemon/queue/DownloadInfo.h b/daemon/queue/DownloadInfo.h index c73afa308..0722a395b 100644 --- a/daemon/queue/DownloadInfo.h +++ b/daemon/queue/DownloadInfo.h @@ -195,6 +195,10 @@ class FileInfo void SetPartialState(EPartialState partialState) { m_partialState = partialState; } uint32 GetCrc() { return m_crc; } void SetCrc(uint32 crc) { m_crc = crc; } + const char* GetHash16k() { return m_hash16k; } + void SetHash16k(const char* hash16k) { m_hash16k = hash16k; } + const char* GetParSetId() { return m_parSetId; } + void SetParSetId(const char* parSetId) { m_parSetId = parSetId; } ServerStatList* GetServerStats() { return &m_serverStats; } @@ -232,6 +236,8 @@ class FileInfo bool m_forceDirectWrite = false; EPartialState m_partialState = psNone; uint32 m_crc = 0; + CString m_hash16k; + CString m_parSetId; static int m_idGen; static int m_idMax; @@ -253,18 +259,27 @@ class CompletedFile cfFailure }; - CompletedFile(int id, const char* fileName, EStatus status, uint32 crc); + CompletedFile(int id, const char* filename, EStatus status, uint32 crc, + bool parFile, const char* hash16k, const char* parSetId); int GetId() { return m_id; } - void SetFileName(const char* fileName) { m_fileName = fileName; } - const char* GetFileName() { return m_fileName; } + void SetFilename(const char* filename) { m_filename = filename; } + const char* GetFilename() { return m_filename; } + bool GetParFile() { return m_parFile; } EStatus GetStatus() { return m_status; } uint32 GetCrc() { return m_crc; } + const char* GetHash16k() { return m_hash16k; } + void SetHash16k(const char* hash16k) { m_hash16k = hash16k; } + const char* GetParSetId() { return m_parSetId; } + void SetParSetId(const char* parSetId) { m_parSetId = parSetId; } private: int m_id; - CString m_fileName; + CString m_filename; EStatus m_status; uint32 m_crc; + bool m_parFile; + CString m_hash16k; + CString m_parSetId; }; typedef std::deque CompletedFileList; @@ -326,63 +341,6 @@ class ScriptStatusList : public ScriptStatusListBase ScriptStatus::EStatus CalcTotalStatus(); }; -class RenameInfo -{ -public: - class FileHash - { - public: - FileHash(const char* filename, const char* hash) : - m_filename(filename), m_hash(hash) {} - const char* GetFilename() { return m_filename; } - const char* GetHash() { return m_hash; } - private: - CString m_filename; - CString m_hash; - }; - - typedef std::deque FileHashList; - - class ParFile - { - public: - ParFile(int id, const char* filename, const char* setId) : - m_id(id), m_filename(filename), m_setId(setId) {} - int GetId() { return m_id; } - const char* GetFilename() { return m_filename; } - const char* GetSetId() { return m_setId; } - bool GetCompleted() { return m_completed; } - void SetCompleted(bool completed) { m_completed = completed; } - bool GetWanted() { return m_wanted; } - void SetWanted(bool wanted) { m_wanted = wanted; } - private: - int m_id; - CString m_filename; - CString m_setId; - bool m_completed = false; - bool m_wanted = false; - }; - - typedef std::deque ParFileList; - - bool GetAllFirst() { return m_allFirst; } - void SetAllFirst(bool allFirst) { m_allFirst = allFirst; } - bool GetWaitingPar() { return m_waitingPar; } - void SetWaitingPar(bool waitingPar) { m_waitingPar = waitingPar; } - bool GetLoadingPar() { return m_loadingPar; } - void SetLoadingPar(bool loadingPar) { m_loadingPar = loadingPar; } - FileHashList* GetArticleHashes() { return &m_articleHashes; } - ParFileList* GetParFiles() { return &m_parFiles; } - void Reset(); - -private: - bool m_allFirst = false; - bool m_waitingPar = false; - bool m_loadingPar = false; - FileHashList m_articleHashes; - ParFileList m_parFiles; -}; - enum EDupeMode { dmScore, @@ -393,7 +351,15 @@ enum EDupeMode class NzbInfo { public: - enum ERenameStatus + enum EDirectRenameStatus + { + tsNone, + tsRunning, + tsFailure, + tsSuccess + }; + + enum EPostRenameStatus { rsNone, rsSkipped, @@ -546,10 +512,12 @@ class NzbInfo void BuildDestDirName(); CString BuildFinalDirName(); CompletedFileList* GetCompletedFiles() { return &m_completedFiles; } - ERenameStatus GetParRenameStatus() { return m_parRenameStatus; } - void SetParRenameStatus(ERenameStatus renameStatus) { m_parRenameStatus = renameStatus; } - ERenameStatus GetRarRenameStatus() { return m_rarRenameStatus; } - void SetRarRenameStatus(ERenameStatus renameStatus) { m_rarRenameStatus = renameStatus; } + void SetDirectRenameStatus(EDirectRenameStatus renameStatus) { m_directRenameStatus = renameStatus; } + EDirectRenameStatus GetDirectRenameStatus() { return m_directRenameStatus; } + EPostRenameStatus GetParRenameStatus() { return m_parRenameStatus; } + void SetParRenameStatus(EPostRenameStatus renameStatus) { m_parRenameStatus = renameStatus; } + EPostRenameStatus GetRarRenameStatus() { return m_rarRenameStatus; } + void SetRarRenameStatus(EPostRenameStatus renameStatus) { m_rarRenameStatus = renameStatus; } EParStatus GetParStatus() { return m_parStatus; } void SetParStatus(EParStatus parStatus) { m_parStatus = parStatus; } EUnpackStatus GetUnpackStatus() { return m_unpackStatus; } @@ -638,7 +606,12 @@ class NzbInfo void SetMessageCount(int messageCount) { m_messageCount = messageCount; } int GetCachedMessageCount() { return m_cachedMessageCount; } GuardedMessageList GuardCachedMessages() { return GuardedMessageList(&m_messages, &m_logMutex); } - RenameInfo* GetRenameInfo() { return &m_renameInfo; } + bool GetAllFirst() { return m_allFirst; } + void SetAllFirst(bool allFirst) { m_allFirst = allFirst; } + bool GetWaitingPar() { return m_waitingPar; } + void SetWaitingPar(bool waitingPar) { m_waitingPar = waitingPar; } + bool GetLoadingPar() { return m_loadingPar; } + void SetLoadingPar(bool loadingPar) { m_loadingPar = loadingPar; } void UpdateCurrentStats(); void UpdateCompletedStats(FileInfo* fileInfo); void UpdateDeletedStats(FileInfo* fileInfo); @@ -680,8 +653,9 @@ class NzbInfo time_t m_maxTime = 0; int m_priority = 0; CompletedFileList m_completedFiles; - ERenameStatus m_parRenameStatus = rsNone; - ERenameStatus m_rarRenameStatus = rsNone; + EDirectRenameStatus m_directRenameStatus = tsNone; + EPostRenameStatus m_parRenameStatus = rsNone; + EPostRenameStatus m_rarRenameStatus = rsNone; EParStatus m_parStatus = psNone; EUnpackStatus m_unpackStatus = usNone; ECleanupStatus m_cleanupStatus = csNone; @@ -728,7 +702,9 @@ class NzbInfo int m_messageCount = 0; int m_cachedMessageCount = 0; int m_feedId = 0; - RenameInfo m_renameInfo; + bool m_allFirst = false; + bool m_waitingPar = false; + bool m_loadingPar = false; static int m_idGen; static int m_idMax; diff --git a/daemon/queue/HistoryCoordinator.cpp b/daemon/queue/HistoryCoordinator.cpp index 09a0f8ad8..2453cae42 100644 --- a/daemon/queue/HistoryCoordinator.cpp +++ b/daemon/queue/HistoryCoordinator.cpp @@ -130,8 +130,8 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb for (FileInfo* fileInfo : nzbInfo->GetFileList()) { nzbInfo->UpdateCompletedStats(fileInfo); - nzbInfo->GetCompletedFiles()->emplace_back(fileInfo->GetId(), - fileInfo->GetFilename(), CompletedFile::cfNone, 0); + nzbInfo->GetCompletedFiles()->emplace_back(fileInfo->GetId(), fileInfo->GetFilename(), + CompletedFile::cfNone, 0, fileInfo->GetParFile(), fileInfo->GetHash16k(), fileInfo->GetParSetId()); } // Cleaning up parked files if par-check was successful or unpack was successful or @@ -169,7 +169,7 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb (completedFile.GetStatus() == CompletedFile::cfPartial && &completedFile == &*nzbInfo->GetCompletedFiles()->rbegin())) { - nzbInfo->PrintMessage(Message::mkDetail, "Parking file %s", completedFile.GetFileName()); + nzbInfo->PrintMessage(Message::mkDetail, "Parking file %s", completedFile.GetFilename()); nzbInfo->SetParkedFileCount(nzbInfo->GetParkedFileCount() + 1); } } @@ -178,6 +178,11 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb nzbInfo->SetRemainingParCount(0); nzbInfo->SetParking(false); + if (nzbInfo->GetDirectRenameStatus() == NzbInfo::tsRunning) + { + nzbInfo->SetDirectRenameStatus(NzbInfo::tsFailure); + } + nzbInfo->PrintMessage(Message::mkInfo, "Collection %s added to history", nzbInfo->GetName()); } @@ -493,6 +498,7 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History nzbInfo->SetParStatus(NzbInfo::psNone); nzbInfo->SetParRenameStatus(NzbInfo::rsNone); nzbInfo->SetRarRenameStatus(NzbInfo::rsNone); + nzbInfo->SetDirectRenameStatus(NzbInfo::tsNone); nzbInfo->SetDownloadedSize(0); nzbInfo->SetDownloadSec(0); nzbInfo->SetPostTotalSec(0); @@ -500,10 +506,12 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History nzbInfo->SetRepairSec(0); nzbInfo->SetUnpackSec(0); nzbInfo->SetExtraParBlocks(0); + nzbInfo->SetAllFirst(false); + nzbInfo->SetWaitingPar(false); + nzbInfo->SetLoadingPar(false); nzbInfo->GetCompletedFiles()->clear(); nzbInfo->GetServerStats()->clear(); nzbInfo->GetCurrentServerStats()->clear(); - nzbInfo->GetRenameInfo()->Reset(); nzbInfo->MoveFileList(newNzbInfo.get()); @@ -588,7 +596,7 @@ void HistoryCoordinator::HistoryRetry(DownloadQueue* downloadQueue, HistoryList: g_DiskState->LoadFileState(fileInfo.get(), g_ServerPool->GetServers(), true) && (resetFailed || fileInfo->GetRemainingSize() > 0)))) { - fileInfo->SetFilename(completedFile.GetFileName()); + fileInfo->SetFilename(completedFile.GetFilename()); fileInfo->SetNzbInfo(nzbInfo); BString<1024> outputFilename("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, fileInfo->GetFilename()); diff --git a/daemon/queue/QueueCoordinator.cpp b/daemon/queue/QueueCoordinator.cpp index 195fb460e..28af9c93d 100644 --- a/daemon/queue/QueueCoordinator.cpp +++ b/daemon/queue/QueueCoordinator.cpp @@ -499,7 +499,9 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f break; } - if (g_Options->GetDirectRename() && !fileInfo->GetNzbInfo()->GetRenameInfo()->GetAllFirst() && + if (g_Options->GetDirectRename() && + fileInfo->GetNzbInfo()->GetDirectRenameStatus() <= NzbInfo::tsRunning && + !fileInfo->GetNzbInfo()->GetAllFirst() && GetNextFirstArticle(fileInfo->GetNzbInfo(), fileInfo, articleInfo)) { return true; @@ -551,6 +553,7 @@ bool QueueCoordinator::GetNextFirstArticle(NzbInfo* nzbInfo, FileInfo* &fileInfo { fileInfo = fileInfo1; articleInfo = article; + nzbInfo->SetDirectRenameStatus(NzbInfo::tsRunning); return true; } } @@ -558,7 +561,7 @@ bool QueueCoordinator::GetNextFirstArticle(NzbInfo* nzbInfo, FileInfo* &fileInfo } // no more files for renaming remained - nzbInfo->GetRenameInfo()->SetAllFirst(true); + nzbInfo->SetAllFirst(true); return false; } @@ -640,9 +643,9 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader) { articleInfo->SetStatus(ArticleInfo::aiUndefined); retry = true; - if (g_Options->GetDirectRename() && articleInfo->GetPartNumber() == 0) + if (articleInfo->GetPartNumber() == 1) { - nzbInfo->GetRenameInfo()->SetAllFirst(false); + nzbInfo->SetAllFirst(false); } } @@ -789,7 +792,8 @@ void QueueCoordinator::DeleteFileInfo(DownloadQueue* downloadQueue, FileInfo* fi completed && fileInfo->GetOutputFilename() ? FileSystem::BaseFileName(fileInfo->GetOutputFilename()) : fileInfo->GetFilename(), fileStatus, - fileStatus == CompletedFile::cfSuccess ? fileInfo->GetCrc() : 0); + fileStatus == CompletedFile::cfSuccess ? fileInfo->GetCrc() : 0, + fileInfo->GetParFile(), fileInfo->GetHash16k(), fileInfo->GetParSetId()); } if (g_Options->GetDirectRename()) @@ -1294,16 +1298,16 @@ void QueueCoordinator::DirectRenameCompleted(DownloadQueue* downloadQueue, NzbIn fileInfo->SetCompletedArticles(0); fileInfo->SetRemainingSize(fileInfo->GetSize() - fileInfo->GetMissedSize()); + // discard temporary files + DiscardTempFiles(fileInfo); + g_DiskState->DiscardFile(fileInfo->GetId(), false, true, false); + fileInfo->SetOutputFilename(nullptr); fileInfo->SetOutputInitialized(false); fileInfo->SetCachedArticles(0); fileInfo->SetPartialChanged(false); fileInfo->SetPartialState(FileInfo::psNone); - // discard temporary files - DiscardTempFiles(fileInfo); - g_DiskState->DiscardFile(fileInfo->GetId(), false, true, false); - if (g_Options->GetSaveQueue() && g_Options->GetServerMode()) { // free up memory used by articles if possible @@ -1338,6 +1342,8 @@ void QueueCoordinator::DirectRenameCompleted(DownloadQueue* downloadQueue, NzbIn *Util::FormatSize(discardedSize), discardedCount); } + nzbInfo->SetDirectRenameStatus(NzbInfo::tsSuccess); + if (g_Options->GetParCheck() != Options::pcForce) { downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupResume, nullptr); @@ -1348,4 +1354,6 @@ void QueueCoordinator::DirectRenameCompleted(DownloadQueue* downloadQueue, NzbIn { downloadQueue->EditEntry(nzbInfo->GetId(), DownloadQueue::eaGroupSortFiles, nullptr); } + + downloadQueue->Save(); } diff --git a/tests/functional/rename/rename_opt2_test.py b/tests/functional/rename/rename_opt2_test.py index f88f3b2c0..974eada4b 100644 --- a/tests/functional/rename/rename_opt2_test.py +++ b/tests/functional/rename/rename_opt2_test.py @@ -20,14 +20,40 @@ def test_rename_obf3(nserv, nzbget): hist = nzbget.download_nzb('obfuscated3.nzb', unpack=True) assert hist['Status'] == 'SUCCESS/UNPACK' -def test_rename_obf1_damaged(nserv, nzbget): +def test_rename_obf1dm(nserv, nzbget): nzb_content = nzbget.load_nzb('obfuscated1.nzb') nzb_content = nzb_content.replace('abc.01?4=300000:100000', 'abc.01?4=300000:100000!0') hist = nzbget.download_nzb('obfuscated1-damaged.nzb', nzb_content, unpack=True) assert hist['Status'] == 'SUCCESS/UNPACK' -def test_rename_obf3_damaged(nserv, nzbget): +def test_rename_obf1dmf(nserv, nzbget): + nzb_content = nzbget.load_nzb('obfuscated1.nzb') + nzb_content = nzb_content.replace('abc.01?1=0:100000', 'abc.01?1=0:100000!0') + hist = nzbget.download_nzb('obfuscated1-damaged-first.nzb', nzb_content, unpack=True) + assert hist['Status'] == 'SUCCESS/UNPACK' + +def test_rename_obf1dmf2(nserv, nzbget): + nzb_content = nzbget.load_nzb('obfuscated1.nzb') + nzb_content = nzb_content.replace('abc.01?1=0:100000', 'abc.01?1=0:100000!0') + nzb_content = nzb_content.replace('abc.00?1=0:108', 'abc.00?1=0:108!0') + hist = nzbget.download_nzb('obfuscated1-damaged-first2.nzb', nzb_content, unpack=True) + assert hist['Status'] == 'SUCCESS/UNPACK' + +def test_rename_obf3dm(nserv, nzbget): nzb_content = nzbget.load_nzb('obfuscated3.nzb') nzb_content = nzb_content.replace('abc.01?17=1600000:100000', 'abc.01?17=1600000:100000!0') hist = nzbget.download_nzb('obfuscated3-damaged.nzb', nzb_content, unpack=True) assert hist['Status'] == 'SUCCESS/UNPACK' + +def test_rename_obf3dmf(nserv, nzbget): + nzb_content = nzbget.load_nzb('obfuscated3.nzb') + nzb_content = nzb_content.replace('abc.01?11=0:100000', 'abc.01?11=0:100000!0') + hist = nzbget.download_nzb('obfuscated3-damaged-first.nzb', nzb_content, unpack=True) + assert hist['Status'] == 'SUCCESS/UNPACK' + +def test_rename_obf3dmf2(nserv, nzbget): + nzb_content = nzbget.load_nzb('obfuscated3.nzb') + nzb_content = nzb_content.replace('abc.01?11=0:100000', 'abc.01?11=0:100000!0') + nzb_content = nzb_content.replace('abc.00?1=0:4704', 'abc.00?1=0:4704!0') + hist = nzbget.download_nzb('obfuscated3-damaged-first2.nzb', nzb_content, unpack=True) + assert hist['Status'] == 'SUCCESS/UNPACK'