Skip to content
This repository has been archived by the owner on Nov 18, 2022. It is now read-only.

Commit

Permalink
#438: hibernating all file infos in one large disk state file
Browse files Browse the repository at this point in the history 
to greatly improve startup time
  • Loading branch information
@hugbug
hugbug committed Sep 2, 2017
1 parent b7c3e82 commit 7b7fb6a
Show file tree
Hide file tree
Showing 4 changed files with 166 additions and 35 deletions.
178 changes: 147 additions & 31 deletions daemon/queue/DiskState.cpp
View file
Expand Up @@ -27,6 +27,10 @@
#include "FileSystem.h"

static const char* FORMATVERSION_SIGNATURE = "nzbget diskstate file version ";
const int DISKSTATE_QUEUE_VERSION = 60;
const int DISKSTATE_FILE_VERSION = 5;
const int DISKSTATE_STATS_VERSION = 3;
const int DISKSTATE_FEEDS_VERSION = 3;

class StateDiskFile : public DiskFile
{
Expand Down Expand Up @@ -265,7 +269,7 @@ bool DiskState::SaveDownloadQueue(DownloadQueue* downloadQueue, bool saveHistory
bool ok = true;

{
StateFile stateFile("queue", 60, true);
StateFile stateFile("queue", DISKSTATE_QUEUE_VERSION, true);
if (!downloadQueue->GetQueue()->empty())
{
StateDiskFile* outfile = stateFile.BeginWrite();
Expand All @@ -288,7 +292,7 @@ bool DiskState::SaveDownloadQueue(DownloadQueue* downloadQueue, bool saveHistory

if (saveHistory)
{
StateFile stateFile("history", 60, true);
StateFile stateFile("history", DISKSTATE_QUEUE_VERSION, true);
if (!downloadQueue->GetHistory()->empty())
{
StateDiskFile* outfile = stateFile.BeginWrite();
Expand Down Expand Up @@ -320,7 +324,7 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers
int formatVersion = 0;

{
StateFile stateFile("queue", 60, true);
StateFile stateFile("queue", DISKSTATE_QUEUE_VERSION, true);
if (stateFile.FileExists())
{
StateDiskFile* infile = stateFile.BeginRead();
Expand Down Expand Up @@ -349,7 +353,7 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers

if (formatVersion == 0 || formatVersion >= 57)
{
StateFile stateFile("history", 60, true);
StateFile stateFile("history", DISKSTATE_QUEUE_VERSION, true);
if (stateFile.FileExists())
{
StateDiskFile* infile = stateFile.BeginRead();
Expand All @@ -361,6 +365,8 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers
}
}

LoadAllFileInfos(downloadQueue);

CleanupQueueDir(downloadQueue);

if (!LoadAllFileStates(downloadQueue, servers)) goto error;
Expand Down Expand Up @@ -819,19 +825,14 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i

std::unique_ptr<FileInfo> fileInfo = std::make_unique<FileInfo>();
fileInfo->SetId(id);

bool res = LoadFile(fileInfo.get(), true, false);
if (res)
fileInfo->SetPaused(paused);
if (formatVersion < 56)
{
fileInfo->SetPaused(paused);
if (formatVersion < 56)
{
fileInfo->SetTime(time);
}
fileInfo->SetExtraPriority((bool)extraPriority);
fileInfo->SetNzbInfo(nzbInfo);
nzbInfo->GetFileList()->Add(std::move(fileInfo));
fileInfo->SetTime(time);
}
fileInfo->SetExtraPriority((bool)extraPriority);
fileInfo->SetNzbInfo(nzbInfo);
nzbInfo->GetFileList()->Add(std::move(fileInfo));
}

return true;
Expand Down Expand Up @@ -884,18 +885,18 @@ bool DiskState::SaveFile(FileInfo* fileInfo)
debug("Saving FileInfo %i to disk", fileInfo->GetId());

BString<100> filename("%i", fileInfo->GetId());
StateFile stateFile(filename, 5, false);
StateFile stateFile(filename, DISKSTATE_FILE_VERSION, false);

StateDiskFile* outfile = stateFile.BeginWrite();
if (!outfile)
{
return false;
}

return SaveFileInfo(fileInfo, *outfile) && stateFile.FinishWrite();
return SaveFileInfo(fileInfo, *outfile, true) && stateFile.FinishWrite();
}

bool DiskState::SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile)
bool DiskState::SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile, bool articles)
{
outfile.PrintLine("%s", fileInfo->GetSubject());
outfile.PrintLine("%s", fileInfo->GetFilename());
Expand All @@ -918,11 +919,14 @@ bool DiskState::SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile)
outfile.PrintLine("%s", *group);
}

outfile.PrintLine("%i", (int)fileInfo->GetArticles()->size());
for (ArticleInfo* articleInfo : fileInfo->GetArticles())
if (articles)
{
outfile.PrintLine("%i,%i", articleInfo->GetPartNumber(), articleInfo->GetSize());
outfile.PrintLine("%s", articleInfo->GetMessageId());
outfile.PrintLine("%i", (int)fileInfo->GetArticles()->size());
for (ArticleInfo* articleInfo : fileInfo->GetArticles())
{
outfile.PrintLine("%i,%i", articleInfo->GetPartNumber(), articleInfo->GetSize());
outfile.PrintLine("%s", articleInfo->GetMessageId());
}
}

return true;
Expand All @@ -938,7 +942,7 @@ bool DiskState::LoadFile(FileInfo* fileInfo, bool fileSummary, bool articles)
debug("Loading FileInfo %i from disk", fileInfo->GetId());

BString<100> filename("%i", fileInfo->GetId());
StateFile stateFile(filename, 5, false);
StateFile stateFile(filename, DISKSTATE_FILE_VERSION, false);

StateDiskFile* infile = stateFile.BeginRead();
if (!infile)
Expand Down Expand Up @@ -999,10 +1003,9 @@ bool DiskState::LoadFileInfo(FileInfo* fileInfo, StateDiskFile& infile, int form
if (fileSummary) fileInfo->GetGroups()->push_back(buf);
}

if (infile.ScanLine("%i", &size) != 1) goto error;

if (articles)
{
if (infile.ScanLine("%i", &size) != 1) goto error;
for (int i = 0; i < size; i++)
{
int PartNumber, PartSize;
Expand Down Expand Up @@ -1030,7 +1033,7 @@ bool DiskState::SaveFileState(FileInfo* fileInfo, bool completed)
debug("Saving FileState %i to disk", fileInfo->GetId());

BString<100> filename("%i%s", fileInfo->GetId(), completed ? "c" : "s");
StateFile stateFile(filename, 5, false);
StateFile stateFile(filename, DISKSTATE_FILE_VERSION, false);

StateDiskFile* outfile = stateFile.BeginWrite();
if (!outfile)
Expand Down Expand Up @@ -1073,7 +1076,7 @@ bool DiskState::LoadFileState(FileInfo* fileInfo, Servers* servers, bool complet
debug("Loading FileInfo %i from disk", fileInfo->GetId());

BString<100> filename("%i%s", fileInfo->GetId(), completed ? "c" : "s");
StateFile stateFile(filename, 5, false);
StateFile stateFile(filename, DISKSTATE_FILE_VERSION, false);

StateDiskFile* infile = stateFile.BeginRead();
if (!infile)
Expand Down Expand Up @@ -1514,7 +1517,7 @@ bool DiskState::SaveFeeds(Feeds* feeds, FeedHistory* feedHistory)
{
debug("Saving feeds state to disk");

StateFile stateFile("feeds", 3, true);
StateFile stateFile("feeds", DISKSTATE_FEEDS_VERSION, true);

if (feeds->empty() && feedHistory->empty())
{
Expand Down Expand Up @@ -1542,7 +1545,7 @@ bool DiskState::LoadFeeds(Feeds* feeds, FeedHistory* feedHistory)
{
debug("Loading feeds state from disk");

StateFile stateFile("feeds", 3, true);
StateFile stateFile("feeds", DISKSTATE_FEEDS_VERSION, true);

if (!stateFile.FileExists())
{
Expand Down Expand Up @@ -1684,6 +1687,119 @@ void DiskState::CalcFileStats(DownloadQueue* downloadQueue, int formatVersion)
}
}

bool DiskState::SaveAllFileInfos(DownloadQueue* downloadQueue)
{
bool ok = true;
StateFile stateFile("files", DISKSTATE_FILE_VERSION, true);
if (!downloadQueue->GetQueue()->empty())
{
StateDiskFile* outfile = stateFile.BeginWrite();
if (!outfile)
{
return false;
}

// save file-infos

int fileCount = 0;
for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
{
fileCount += nzbInfo->GetFileList()->size();
}
outfile->PrintLine("%i", fileCount);

for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
{
for (FileInfo* fileInfo : nzbInfo->GetFileList())
{
outfile->PrintLine("%i", fileInfo->GetId());
SaveFileInfo(fileInfo, *outfile, false);
}
}

// now rename to dest file name
ok = stateFile.FinishWrite();
}
else
{
stateFile.Discard();
}

return ok;
}

bool DiskState::LoadAllFileInfos(DownloadQueue* downloadQueue)
{
if (downloadQueue->GetQueue()->empty())
{
return true;
}

StateFile stateFile("files", DISKSTATE_FILE_VERSION, false);
StateDiskFile* infile = nullptr;
bool useHibernate = false;

if (stateFile.FileExists())
{
infile = stateFile.BeginRead();
useHibernate = infile != nullptr;
if (useHibernate)
{
int fileCount = 0;
for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
{
fileCount += nzbInfo->GetFileList()->size();
}
int size = 0;
useHibernate = infile->ScanLine("%i", &size) == 1 && size == fileCount;
}
if (!useHibernate)
{
stateFile.Discard();
}
}

for (NzbInfo* nzbInfo : downloadQueue->GetQueue())
{
RawFileList brokenFileInfos;

for (FileInfo* fileInfo : nzbInfo->GetFileList())
{
bool res = false;
if (useHibernate)
{
int id = 0;
infile->ScanLine("%i", &id);
if (id == fileInfo->GetId())
{
res = LoadFileInfo(fileInfo, *infile, stateFile.GetFileVersion(), true, false);
}
}
if (!res)
{
res = LoadFile(fileInfo, true, false);
}
if (!res)
{
brokenFileInfos.push_back(fileInfo);
}
}

for (FileInfo* fileInfo : brokenFileInfos)
{
nzbInfo->GetFileList()->Remove(fileInfo);
}
}

return true;
}

void DiskState::DiscardQuickFileInfos()
{
StateFile stateFile("files", DISKSTATE_FILE_VERSION, false);
stateFile.Discard();
}

bool DiskState::LoadAllFileStates(DownloadQueue* downloadQueue, Servers* servers)
{
BString<1024> cacheFlagFilename("%s%c%s", g_Options->GetQueueDir(), PATH_SEPARATOR, "acache");
Expand Down Expand Up @@ -1731,7 +1847,7 @@ bool DiskState::SaveStats(Servers* servers, ServerVolumes* serverVolumes)
{
debug("Saving stats to disk");

StateFile stateFile("stats", 3, true);
StateFile stateFile("stats", DISKSTATE_STATS_VERSION, true);

if (servers->empty())
{
Expand Down Expand Up @@ -1759,7 +1875,7 @@ bool DiskState::LoadStats(Servers* servers, ServerVolumes* serverVolumes, bool*
{
debug("Loading stats from disk");

StateFile stateFile("stats", 3, true);
StateFile stateFile("stats", DISKSTATE_STATS_VERSION, true);

if (!stateFile.FileExists())
{
Expand Down
7 changes: 5 additions & 2 deletions daemon/queue/DiskState.h
View file
@@ -1,7 +1,7 @@
/*
* This file is part of nzbget. See <http://nzbget.net>.
*
* Copyright (C) 2007-2016 Andrey Prygunkov <hugbug@users.sourceforge.net>
* Copyright (C) 2007-2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
Expand Down Expand Up @@ -38,6 +38,8 @@ class DiskState
bool LoadDownloadQueue(DownloadQueue* downloadQueue, Servers* servers);
bool SaveFile(FileInfo* fileInfo);
bool LoadFile(FileInfo* fileInfo, bool fileSummary, bool articles);
bool SaveAllFileInfos(DownloadQueue* downloadQueue);
void DiscardQuickFileInfos();
bool SaveFileState(FileInfo* fileInfo, bool completed);
bool LoadFileState(FileInfo* fileInfo, Servers* servers, bool completed);
bool LoadArticles(FileInfo* fileInfo);
Expand All @@ -55,7 +57,7 @@ class DiskState
void LoadNzbMessages(int nzbId, MessageList* messages);

private:
bool SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile);
bool SaveFileInfo(FileInfo* fileInfo, StateDiskFile& outfile, bool articles);
bool LoadFileInfo(FileInfo* fileInfo, StateDiskFile& outfile, int formatVersion, bool fileSummary, bool articles);
bool SaveFileState(FileInfo* fileInfo, StateDiskFile& outfile, bool completed);
bool LoadFileState(FileInfo* fileInfo, Servers* servers, StateDiskFile& infile, int formatVersion, bool completed);
Expand All @@ -76,6 +78,7 @@ class DiskState
bool SaveVolumeStat(ServerVolumes* serverVolumes, StateDiskFile& outfile);
bool LoadVolumeStat(Servers* servers, ServerVolumes* serverVolumes, StateDiskFile& infile, int formatVersion);
void CalcFileStats(DownloadQueue* downloadQueue, int formatVersion);
bool LoadAllFileInfos(DownloadQueue* downloadQueue);
bool LoadAllFileStates(DownloadQueue* downloadQueue, Servers* servers);
void SaveServerStats(ServerStatList* serverStatList, StateDiskFile& outfile);
bool LoadServerStats(ServerStatList* serverStatList, Servers* servers, StateDiskFile& infile);
Expand Down
14 changes: 12 additions & 2 deletions daemon/queue/QueueCoordinator.cpp
View file
Expand Up @@ -61,6 +61,7 @@ void QueueCoordinator::CoordinatorDownloadQueue::Save()
if (g_Options->GetSaveQueue() && g_Options->GetServerMode())
{
g_DiskState->SaveDownloadQueue(this, m_historyChanged);
m_stateChanged = true;
}

for (NzbInfo* nzbInfo : GetQueue())
Expand Down Expand Up @@ -262,6 +263,7 @@ void QueueCoordinator::Run()

WaitJobs();
SaveAllPartialState();
SaveAllFileState();

debug("Exiting QueueCoordinator-loop");
}
Expand Down Expand Up @@ -464,8 +466,7 @@ bool QueueCoordinator::GetNextArticle(DownloadQueue* downloadQueue, FileInfo* &f
// if the file doesn't have any articles left for download, we store that fact and search again,
// ignoring all files which were previously marked as not having any articles.

// special case: if the file has ExtraPriority-flag set, it has the highest priority and the
// Paused-flag is ignored.
// special case: if the file has ExtraPriority-flag set, it has the highest priority.

//debug("QueueCoordinator::GetNextArticle()");

Expand Down Expand Up @@ -906,6 +907,15 @@ void QueueCoordinator::LoadPartialState(FileInfo* fileInfo)
}
}

void QueueCoordinator::SaveAllFileState()
{
if (g_Options->GetSaveQueue() && g_Options->GetServerMode() && m_downloadQueue.m_stateChanged)
{
GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
g_DiskState->SaveAllFileInfos(downloadQueue);
}
}

void QueueCoordinator::CheckHealth(DownloadQueue* downloadQueue, FileInfo* fileInfo)
{
if (g_Options->GetHealthCheck() == Options::hcNone ||
Expand Down

0 comments on commit 7b7fb6a

Please sign in to comment.