Permalink
Browse files

[videoinfoscanner] recursive fasthash for enumerating series folders

  • Loading branch information...
1 parent 4c5b735 commit d0047b7812a7b69ca1d939a596df9b1ec67b4738 @mkortstiege mkortstiege committed Jun 6, 2014
Showing with 93 additions and 15 deletions.
  1. +80 −14 xbmc/video/VideoInfoScanner.cpp
  2. +13 −1 xbmc/video/VideoInfoScanner.h
@@ -641,7 +641,8 @@ namespace VIDEO
{
// enumerate episodes
EPISODELIST files;
- EnumerateSeriesFolder(item, files);
+ if (!EnumerateSeriesFolder(item, files))
+ return INFO_HAVE_ALREADY;
if (files.size() == 0) // no update or no files
return INFO_NOT_NEEDED;
@@ -653,30 +654,58 @@ namespace VIDEO
return OnProcessSeriesFolder(files, scraper, useLocal, showInfo, progress);
}
- void CVideoInfoScanner::EnumerateSeriesFolder(CFileItem* item, EPISODELIST& episodeList)
+ bool CVideoInfoScanner::EnumerateSeriesFolder(CFileItem* item, EPISODELIST& episodeList)
{
CFileItemList items;
+ CStdStringArray regexps = g_advancedSettings.m_tvshowExcludeFromScanRegExps;
+
+ bool bSkip = false;
if (item->m_bIsFolder)
{
- CUtil::GetRecursiveListing(item->GetPath(), items, g_advancedSettings.m_videoExtensions, DIR_FLAG_DEFAULTS);
CStdString hash, dbHash;
- int numFilesInFolder = GetPathHash(items, hash);
+ hash = GetRecursiveFastHash(item->GetPath(), regexps);
+ if (m_database.GetPathHash(item->GetPath(), dbHash) && !hash.empty() && dbHash == hash)
+ {
+ // fast hashes match - no need to process anything
+ bSkip = true;
+ }
- if (m_database.GetPathHash(item->GetPath(), dbHash) && dbHash == hash)
+ // fast hash cannot be computed or we need to rescan. fetch the listing.
+ if (!bSkip)
{
- m_currentItem += numFilesInFolder;
+ int flags = DIR_FLAG_DEFAULTS;
+ if (!hash.empty())
+ flags |= DIR_FLAG_NO_FILE_INFO;
- // update our dialog with our progress
- if (m_handle)
+ CUtil::GetRecursiveListing(item->GetPath(), items, g_advancedSettings.m_videoExtensions, flags);
+
+ // fast hash failed - compute slow one
+ if (hash.empty())
{
- if (m_itemCount>0)
- m_handle->SetPercentage(m_currentItem*100.f/m_itemCount);
+ GetPathHash(items, hash);
+ if (dbHash == hash)
+ {
+ // slow hashes match - no need to process anything
+ bSkip = true;
+ }
+ }
+ }
+ if (bSkip)
+ {
+ CLog::Log(LOGDEBUG, "VideoInfoScanner: Skipping dir '%s' due to no change", CURL::GetRedacted(item->GetPath()).c_str());
+ // update our dialog with our progress
+ if (m_handle)
OnDirectoryScanned(item->GetPath());
- }
- return;
+ return false;
}
+
+ if (dbHash.empty())
+ CLog::Log(LOGDEBUG, "VideoInfoScanner: Scanning dir '%s' as not in the database", CURL::GetRedacted(item->GetPath()).c_str());
+ else
+ CLog::Log(LOGDEBUG, "VideoInfoScanner: Rescanning dir '%s' due to change (%s != %s)", CURL::GetRedacted(item->GetPath()).c_str(), dbHash.c_str(), hash.c_str());
+
m_pathsToClean.insert(m_database.GetPathId(item->GetPath()));
m_database.GetPathsForTvShow(m_database.GetTvShowId(item->GetPath()), m_pathsToClean);
item->SetProperty("hash", hash);
@@ -735,8 +764,6 @@ namespace VIDEO
}
// enumerate
- CStdStringArray regexps = g_advancedSettings.m_tvshowExcludeFromScanRegExps;
-
for (int i=0;i<items.Size();++i)
{
if (items[i]->m_bIsFolder)
@@ -762,6 +789,7 @@ namespace VIDEO
if (!EnumerateEpisodeItem(items[i].get(), episodeList))
CLog::Log(LOGDEBUG, "VideoInfoScanner: Could not enumerate file %s", CURL::GetRedacted(CURL::Decode(items[i]->GetPath())).c_str());
}
+ return true;
}
bool CVideoInfoScanner::ProcessItemByVideoInfoTag(const CFileItem *item, EPISODELIST &episodeList)
@@ -1698,6 +1726,44 @@ namespace VIDEO
return "";
}
+ CStdString CVideoInfoScanner::GetRecursiveFastHash(const CStdString &directory, const CStdStringArray &excludes) const
+ {
+ CFileItemList items;
+ items.Add(CFileItemPtr(new CFileItem(directory, true)));
+ CUtil::GetRecursiveDirsListing(directory, items, DIR_FLAG_NO_FILE_DIRS | DIR_FLAG_NO_FILE_INFO);
+
+ XBMC::XBMC_MD5 md5state;
+
+ if (excludes.size())
+ md5state.append(StringUtils::JoinString(excludes, "|"));
+
+ int64_t time = 0;
+ for (int i=0; i < items.Size(); ++i)
+ {
+ int64_t stat_time = 0;
+ struct __stat64 buffer;
+ if (XFILE::CFile::Stat(items[i]->GetPath(), &buffer) == 0)
+ {
+ // TODO: some filesystems may return the mtime/ctime inline, in which case this is
+ // unnecessarily expensive. Consider supporting Stat() in our directory cache?
+ stat_time = buffer.st_mtime ? buffer.st_mtime : buffer.st_ctime;
+ time += stat_time;
+ }
+
+ if (!stat_time)
+ return "";
+ }
+
+ if (time)
+ {
+ md5state.append((unsigned char *)&time, sizeof(time));
+ CStdString pathHash;
+ md5state.getDigest(pathHash);
+ return pathHash;
+ }
+ return "";
+ }
+
void CVideoInfoScanner::GetSeasonThumbs(const CVideoInfoTag &show, map<int, map<string, string> > &seasonArt, const vector<string> &artTypes, bool useLocal)
{
bool lookForThumb = find(artTypes.begin(), artTypes.end(), "thumb") == artTypes.end();
@@ -193,6 +193,18 @@ namespace VIDEO
*/
CStdString GetFastHash(const CStdString &directory, const CStdStringArray &excludes) const;
+ /*! \brief Retrieve a "fast" hash of the given directory recursively (if available)
+ Performs a stat() on the directory, and uses modified time to create a "fast"
+ hash of each folder. If no modified time is available, the create time is used,
+ and if neither are available, an empty hash is returned.
+ In case exclude from scan expressions are present, the string array will be appended
+ to the md5 hash to ensure we're doing a re-scan whenever the user modifies those.
+ \param directory folder to hash (recursively)
+ \param excludes string array of exclude expressions
+ \return the md5 hash of the folder
+ */
+ CStdString GetRecursiveFastHash(const CStdString &directory, const CStdStringArray &excludes) const;
+
/*! \brief Decide whether a folder listing could use the "fast" hash
Fast hashing can be done whenever the folder contains no scannable subfolders, as the
fast hash technique uses modified time to determine when folder content changes, which
@@ -216,7 +228,7 @@ namespace VIDEO
*/
INFO_RET OnProcessSeriesFolder(EPISODELIST& files, const ADDON::ScraperPtr &scraper, bool useLocal, const CVideoInfoTag& showInfo, CGUIDialogProgress* pDlgProgress = NULL);
- void EnumerateSeriesFolder(CFileItem* item, EPISODELIST& episodeList);
+ bool EnumerateSeriesFolder(CFileItem* item, EPISODELIST& episodeList);
bool EnumerateEpisodeItem(const CFileItem *item, EPISODELIST& episodeList);
bool ProcessItemByVideoInfoTag(const CFileItem *item, EPISODELIST &episodeList);

0 comments on commit d0047b7

Please sign in to comment.