#541: better duplicate handling of urls

daemon/postprocess/PrePostProcessor.cpp

@@ -381,6 +381,7 @@ void PrePostProcessor::NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo
 
 void PrePostProcessor::NzbCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, bool saveQueue)
 {
+	bool downloadDupe = nzbInfo->GetDupeHint() == NzbInfo::dhRedownloadAuto;
 	bool addToHistory = g_Options->GetKeepHistory() > 0 && !nzbInfo->GetAvoidHistory();
 	if (addToHistory)
 	{
@@ -394,7 +395,8 @@ void PrePostProcessor::NzbCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbIn
 		(nzbInfo->GetDeleteStatus() == NzbInfo::dsNone ||
 		 nzbInfo->GetDeleteStatus() == NzbInfo::dsHealth ||
 		 nzbInfo->GetDeleteStatus() == NzbInfo::dsBad ||
-		 nzbInfo->GetDeleteStatus() == NzbInfo::dsScan))
+		 nzbInfo->GetDeleteStatus() == NzbInfo::dsScan ||
+		 (nzbInfo->GetDeleteStatus() == NzbInfo::dsCopy && downloadDupe)))
 	{
 		g_DupeCoordinator->NzbCompleted(downloadQueue, nzbInfo);
 		needSave = true;

daemon/queue/DiskState.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2018 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -27,7 +27,7 @@
 #include "FileSystem.h"
 
 static const char* FORMATVERSION_SIGNATURE = "nzbget diskstate file version ";
-const int DISKSTATE_QUEUE_VERSION = 60;
+const int DISKSTATE_QUEUE_VERSION = 61;
 const int DISKSTATE_FILE_VERSION = 5;
 const int DISKSTATE_STATS_VERSION = 3;
 const int DISKSTATE_FEEDS_VERSION = 3;
@@ -464,7 +464,7 @@ void DiskState::SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile)
 	outfile.PrintLine("%i,%i,%i", nzbInfo->GetTotalArticles(), nzbInfo->GetSuccessArticles(), nzbInfo->GetFailedArticles());
 
 	outfile.PrintLine("%s", nzbInfo->GetDupeKey());
-	outfile.PrintLine("%i,%i", (int)nzbInfo->GetDupeMode(), nzbInfo->GetDupeScore());
+	outfile.PrintLine("%i,%i,%i", (int)nzbInfo->GetDupeMode(), nzbInfo->GetDupeScore(), (int)nzbInfo->GetDupeHint());
 
 	Util::SplitInt64(nzbInfo->GetDownloadedSize(), &High1, &Low1);
 	outfile.PrintLine("%u,%u,%i,%i,%i,%i,%i", High1, Low1, nzbInfo->GetDownloadSec(), nzbInfo->GetPostTotalSec(),
@@ -690,10 +690,19 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i
 	if (!infile.ReadLine(buf, sizeof(buf))) goto error;
 	nzbInfo->SetDupeKey(buf);
 
-	int dupeMode, dupeScore;
-	if (infile.ScanLine("%i,%i", &dupeMode, &dupeScore) != 2) goto error;
+	int dupeMode, dupeScore, dupeHint;
+	dupeHint = 0; //clang requires initialization in a separate line (due to goto statements)
+	if (formatVersion >= 61)
+	{
+		if (infile.ScanLine("%i,%i,%i", &dupeMode, &dupeScore, &dupeHint) != 3) goto error;
+	}
+	else
+	{
+		if (infile.ScanLine("%i,%i", &dupeMode, &dupeScore) != 2) goto error;
+	}
 	nzbInfo->SetDupeMode((EDupeMode)dupeMode);
 	nzbInfo->SetDupeScore(dupeScore);
+	nzbInfo->SetDupeMode((EDupeMode)dupeHint);
 
 	if (formatVersion >= 48)
 	{

daemon/queue/DownloadInfo.h

@@ -2,7 +2,7 @@
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
  *  Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2018 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -446,6 +446,13 @@ class NzbInfo
 		nkUrl
 	};
 
+	enum EDupeHint
+	{
+		dhNone,
+		dhRedownloadManual,
+		dhRedownloadAuto
+	};
+
 	int GetId() { return m_id; }
 	void SetId(int id);
 	static void ResetGenId(bool max);
@@ -580,6 +587,8 @@ class NzbInfo
 	void SetDupeScore(int dupeScore) { m_dupeScore = dupeScore; }
 	EDupeMode GetDupeMode() { return m_dupeMode; }
 	void SetDupeMode(EDupeMode dupeMode) { m_dupeMode = dupeMode; }
+	EDupeHint GetDupeHint() { return m_dupeHint; }
+	void SetDupeHint(EDupeHint dupeHint) { m_dupeHint = dupeHint; }
 	uint32 GetFullContentHash() { return m_fullContentHash; }
 	void SetFullContentHash(uint32 fullContentHash) { m_fullContentHash = fullContentHash; }
 	uint32 GetFilteredContentHash() { return m_filteredContentHash; }
@@ -698,6 +707,7 @@ class NzbInfo
 	CString m_dupeKey = "";
 	int m_dupeScore = 0;
 	EDupeMode m_dupeMode = dmScore;
+	EDupeHint m_dupeHint = dhNone;
 	uint32 m_fullContentHash = 0;
 	uint32 m_filteredContentHash = 0;
 	FileList m_fileList;

daemon/queue/DupeCoordinator.cpp

@@ -202,6 +202,12 @@ void DupeCoordinator::NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 		}
 	}
 
+	if (!sameContent && nzbInfo->GetDupeHint() != NzbInfo::dhNone)
+	{
+		// dupe check when "download again" URLs: checking same content only
+		return;
+	}
+
 	if (!sameContent && !good && nzbInfo->GetDupeMode() == dmScore)
 	{
 		// nzb-files having success-duplicates in recent history (with different content) are added to history for backup
@@ -238,7 +244,7 @@ void DupeCoordinator::NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo)
 				sameContent ? "exactly same content" : good ? "good status" : "success status");
 		}
 
-		if (nzbInfo->GetFeedId())
+		if (nzbInfo->GetFeedId() && nzbInfo->GetDupeHint() == NzbInfo::dhNone)
 		{
 			warn("%s", *message);
 			// Flag saying QueueCoordinator to skip nzb-file
@@ -398,6 +404,7 @@ void DupeCoordinator::ReturnBestDupe(DownloadQueue* downloadQueue, NzbInfo* nzbI
 	if (historyDupe)
 	{
 		info("Found duplicate %s for %s", historyDupe->GetNzbInfo()->GetName(), nzbName);
+		historyDupe->GetNzbInfo()->SetDupeHint(NzbInfo::dhRedownloadAuto);
 		g_HistoryCoordinator->Redownload(downloadQueue, historyDupe);
 	}
 }

daemon/queue/HistoryCoordinator.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2018 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -183,6 +183,8 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb
 		nzbInfo->SetDirectRenameStatus(NzbInfo::tsFailure);
 	}
 
+	nzbInfo->SetDupeHint(NzbInfo::dhNone);
+
 	nzbInfo->PrintMessage(Message::mkInfo, "Collection %s added to history", nzbInfo->GetName());
 }
 
@@ -433,6 +435,7 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History
 		historyInfo->DiscardNzbInfo();
 		nzbInfo->SetUrlStatus(NzbInfo::lsNone);
 		nzbInfo->SetDeleteStatus(NzbInfo::dsNone);
+		nzbInfo->SetDupeHint(nzbInfo->GetDupeHint() == NzbInfo::dhNone ? NzbInfo::dhRedownloadManual : nzbInfo->GetDupeHint());
 		downloadQueue->GetQueue()->Add(std::unique_ptr<NzbInfo>(nzbInfo), true);
 		downloadQueue->GetHistory()->erase(itHistory);
 		return;

daemon/queue/Scanner.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <http://nzbget.net>.
  *
- *  Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *  Copyright (C) 2007-2018 Andrey Prygunkov <hugbug@users.sourceforge.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -438,6 +438,7 @@ bool Scanner::AddFileToQueue(const char* filename, const char* nzbName, const ch
 		nzbInfo->SetUrl(urlInfo->GetUrl());
 		nzbInfo->SetUrlStatus(urlInfo->GetUrlStatus());
 		nzbInfo->SetFeedId(urlInfo->GetFeedId());
+		nzbInfo->SetDupeHint(urlInfo->GetDupeHint());
 	}
 
 	if (nzbFile.GetPassword())