Permalink
Browse files

Merge pull request #4837 from mkortstiege/videoinfoscanner

videoinfoscanner optimizations
  • Loading branch information...
mkortstiege committed Jun 10, 2014
2 parents a44a60f + 6bef2aa commit 6afb499a211a0e3ebd9473a3a1425eebebb67c3f
View
@@ -3631,7 +3631,7 @@ bool CApplication::PlayMedia(const CFileItem& item, int iPlaylist)
if (item.IsSmartPlayList())
{
CFileItemList items;
- CUtil::GetRecursiveListing(item.GetPath(), items, "");
+ CUtil::GetRecursiveListing(item.GetPath(), items, "", DIR_FLAG_NO_FILE_DIRS);
if (items.Size())
{
CSmartPlaylist smartpl;
@@ -412,7 +412,7 @@ void CApplicationMessenger::ProcessMessage(ThreadMessage *pMsg)
else
URIUtils::CreateArchivePath(strPath, "rar", pMsg->strParam.c_str(), "");
- CUtil::GetRecursiveListing(strPath, items, g_advancedSettings.m_pictureExtensions);
+ CUtil::GetRecursiveListing(strPath, items, g_advancedSettings.m_pictureExtensions, XFILE::DIR_FLAG_NO_FILE_DIRS);
if (items.Size() > 0)
{
pSlideShow->Reset();
View
@@ -1363,32 +1363,29 @@ void CUtil::DeleteDirectoryCache(const CStdString &prefix)
}
-void CUtil::GetRecursiveListing(const CStdString& strPath, CFileItemList& items, const CStdString& strMask, bool bUseFileDirectories)
+void CUtil::GetRecursiveListing(const CStdString& strPath, CFileItemList& items, const CStdString& strMask, unsigned int flags /* = DIR_FLAG_DEFAULTS */)
{
CFileItemList myItems;
- int flags = DIR_FLAG_DEFAULTS;
- if (!bUseFileDirectories)
- flags |= DIR_FLAG_NO_FILE_DIRS;
CDirectory::GetDirectory(strPath,myItems,strMask,flags);
for (int i=0;i<myItems.Size();++i)
{
if (myItems[i]->m_bIsFolder)
- CUtil::GetRecursiveListing(myItems[i]->GetPath(),items,strMask,bUseFileDirectories);
+ CUtil::GetRecursiveListing(myItems[i]->GetPath(),items,strMask,flags);
else
items.Add(myItems[i]);
}
}
-void CUtil::GetRecursiveDirsListing(const CStdString& strPath, CFileItemList& item)
+void CUtil::GetRecursiveDirsListing(const CStdString& strPath, CFileItemList& item, unsigned int flags /* = DIR_FLAG_DEFAULTS */)
{
CFileItemList myItems;
- CDirectory::GetDirectory(strPath,myItems,"",DIR_FLAG_NO_FILE_DIRS);
+ CDirectory::GetDirectory(strPath,myItems,"",flags);
for (int i=0;i<myItems.Size();++i)
{
if (myItems[i]->m_bIsFolder && !myItems[i]->GetPath().Equals(".."))
{
item.Add(myItems[i]);
- CUtil::GetRecursiveDirsListing(myItems[i]->GetPath(),item);
+ CUtil::GetRecursiveDirsListing(myItems[i]->GetPath(),item,flags);
}
}
}
View
@@ -147,8 +147,8 @@ class CUtil
static CStdString VideoPlaylistsLocation();
static void GetSkinThemes(std::vector<CStdString>& vecTheme);
- static void GetRecursiveListing(const CStdString& strPath, CFileItemList& items, const CStdString& strMask, bool bUseFileDirectories=false);
- static void GetRecursiveDirsListing(const CStdString& strPath, CFileItemList& items);
+ static void GetRecursiveListing(const CStdString& strPath, CFileItemList& items, const CStdString& strMask, unsigned int flags = 0 /* DIR_FLAG_DEFAULTS */);
+ static void GetRecursiveDirsListing(const CStdString& strPath, CFileItemList& items, unsigned int flags = 0 /* DIR_FLAG_DEFAULTS */);
static void ForceForwardSlashes(CStdString& strPath);
static double AlbumRelevance(const CStdString& strAlbumTemp1, const CStdString& strAlbum1, const CStdString& strArtistTemp1, const CStdString& strArtist1);
@@ -210,6 +210,7 @@ void CAdvancedSettings::Initialize()
m_moviesExcludeFromScanRegExps.clear();
m_moviesExcludeFromScanRegExps.push_back("-trailer");
m_moviesExcludeFromScanRegExps.push_back("[!-._ \\\\/]sample[-._ \\\\/]");
+ m_moviesExcludeFromScanRegExps.push_back("[\\/](proof|subs)[\\/]");
m_tvshowExcludeFromScanRegExps.push_back("[!-._ \\\\/]sample[-._ \\\\/]");
m_folderStackRegExps.clear();
@@ -263,7 +263,7 @@ namespace VIDEO
m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(str), info->Name().c_str()));
}
- CStdString fastHash = GetFastHash(strDirectory);
+ CStdString fastHash = GetFastHash(strDirectory, regexps);
if (m_database.GetPathHash(strDirectory, dbHash) && !fastHash.empty() && fastHash == dbHash)
{ // fast hashes match - no need to process anything
CLog::Log(LOGDEBUG, "VideoInfoScanner: Skipping dir '%s' due to no change (fasthash)", CURL::GetRedacted(strDirectory).c_str());
@@ -297,7 +297,7 @@ namespace VIDEO
OnDirectoryScanned(strDirectory);
}
// update the hash to a fast hash if needed
- if (CanFastHash(items) && !fastHash.empty())
+ if (CanFastHash(items, regexps) && !fastHash.empty())
hash = fastHash;
}
}
@@ -641,7 +641,8 @@ namespace VIDEO
{
// enumerate episodes
EPISODELIST files;
- EnumerateSeriesFolder(item, files);
+ if (!EnumerateSeriesFolder(item, files))
+ return INFO_HAVE_ALREADY;
if (files.size() == 0) // no update or no files
return INFO_NOT_NEEDED;
@@ -653,30 +654,67 @@ namespace VIDEO
return OnProcessSeriesFolder(files, scraper, useLocal, showInfo, progress);
}
- void CVideoInfoScanner::EnumerateSeriesFolder(CFileItem* item, EPISODELIST& episodeList)
+ bool CVideoInfoScanner::EnumerateSeriesFolder(CFileItem* item, EPISODELIST& episodeList)
{
CFileItemList items;
+ CStdStringArray regexps = g_advancedSettings.m_tvshowExcludeFromScanRegExps;
+
+ bool bSkip = false;
if (item->m_bIsFolder)
{
- CUtil::GetRecursiveListing(item->GetPath(), items, g_advancedSettings.m_videoExtensions, true);
+ /*
+ * Note: DoScan() will not remove this path as it's not recursing for tvshows.
+ * Remove this path from the list we're processing in order to avoid hitting
+ * it twice in the main loop.
+ */
+ set<CStdString>::iterator it = m_pathsToScan.find(item->GetPath());
+ if (it != m_pathsToScan.end())
+ m_pathsToScan.erase(it);
+
CStdString hash, dbHash;
- int numFilesInFolder = GetPathHash(items, hash);
+ hash = GetRecursiveFastHash(item->GetPath(), regexps);
+ if (m_database.GetPathHash(item->GetPath(), dbHash) && !hash.empty() && dbHash == hash)
+ {
+ // fast hashes match - no need to process anything
+ bSkip = true;
+ }
- if (m_database.GetPathHash(item->GetPath(), dbHash) && dbHash == hash)
+ // fast hash cannot be computed or we need to rescan. fetch the listing.
+ if (!bSkip)
{
- m_currentItem += numFilesInFolder;
+ int flags = DIR_FLAG_DEFAULTS;
+ if (!hash.empty())
+ flags |= DIR_FLAG_NO_FILE_INFO;
- // update our dialog with our progress
- if (m_handle)
+ CUtil::GetRecursiveListing(item->GetPath(), items, g_advancedSettings.m_videoExtensions, flags);
+
+ // fast hash failed - compute slow one
+ if (hash.empty())
{
- if (m_itemCount>0)
- m_handle->SetPercentage(m_currentItem*100.f/m_itemCount);
+ GetPathHash(items, hash);
+ if (dbHash == hash)
+ {
+ // slow hashes match - no need to process anything
+ bSkip = true;
+ }
+ }
+ }
+ if (bSkip)
+ {
+ CLog::Log(LOGDEBUG, "VideoInfoScanner: Skipping dir '%s' due to no change", CURL::GetRedacted(item->GetPath()).c_str());
+ // update our dialog with our progress
+ if (m_handle)
OnDirectoryScanned(item->GetPath());
- }
- return;
+ return false;
}
+
+ if (dbHash.empty())
+ CLog::Log(LOGDEBUG, "VideoInfoScanner: Scanning dir '%s' as not in the database", CURL::GetRedacted(item->GetPath()).c_str());
+ else
+ CLog::Log(LOGDEBUG, "VideoInfoScanner: Rescanning dir '%s' due to change (%s != %s)", CURL::GetRedacted(item->GetPath()).c_str(), dbHash.c_str(), hash.c_str());
+
m_pathsToClean.insert(m_database.GetPathId(item->GetPath()));
m_database.GetPathsForTvShow(m_database.GetTvShowId(item->GetPath()), m_pathsToClean);
item->SetProperty("hash", hash);
@@ -735,8 +773,6 @@ namespace VIDEO
}
// enumerate
- CStdStringArray regexps = g_advancedSettings.m_tvshowExcludeFromScanRegExps;
-
for (int i=0;i<items.Size();++i)
{
if (items[i]->m_bIsFolder)
@@ -762,6 +798,7 @@ namespace VIDEO
if (!EnumerateEpisodeItem(items[i].get(), episodeList))
CLog::Log(LOGDEBUG, "VideoInfoScanner: Could not enumerate file %s", CURL::GetRedacted(CURL::Decode(items[i]->GetPath())).c_str());
}
+ return true;
}
bool CVideoInfoScanner::ProcessItemByVideoInfoTag(const CFileItem *item, EPISODELIST &episodeList)
@@ -1664,26 +1701,74 @@ namespace VIDEO
return count;
}
- bool CVideoInfoScanner::CanFastHash(const CFileItemList &items) const
+ bool CVideoInfoScanner::CanFastHash(const CFileItemList &items, const CStdStringArray &excludes) const
{
- // TODO: Probably should account for excluded folders here (eg samples), though that then
- // introduces possible problems if the user then changes the exclude regexps and
- // expects excluded folders that are inside a fast-hashed folder to then be picked
- // up. The chances that the user has a folder which contains only excluded folders
- // where some of those folders should be scanned recursively is pretty small.
- return items.GetFolderCount() == 0;
+ for (int i = 0; i < items.Size(); ++i)
+ {
+ if (items[i]->m_bIsFolder && !CUtil::ExcludeFileOrFolder(items[i]->GetPath(), excludes))
+ return false;
+ }
+ return true;
}
- CStdString CVideoInfoScanner::GetFastHash(const CStdString &directory) const
+ CStdString CVideoInfoScanner::GetFastHash(const CStdString &directory, const CStdStringArray &excludes) const
{
+ XBMC::XBMC_MD5 md5state;
+
+ if (excludes.size())
+ md5state.append(StringUtils::JoinString(excludes, "|"));
+
struct __stat64 buffer;
if (XFILE::CFile::Stat(directory, &buffer) == 0)
{
int64_t time = buffer.st_mtime;
if (!time)
time = buffer.st_ctime;
if (time)
- return StringUtils::Format("fast%"PRId64, time);
+ {
+ md5state.append((unsigned char *)&time, sizeof(time));
+ CStdString pathHash;
+ md5state.getDigest(pathHash);
+ return pathHash;
+ }
+ }
+ return "";
+ }
+
+ CStdString CVideoInfoScanner::GetRecursiveFastHash(const CStdString &directory, const CStdStringArray &excludes) const
+ {
+ CFileItemList items;
+ items.Add(CFileItemPtr(new CFileItem(directory, true)));
+ CUtil::GetRecursiveDirsListing(directory, items, DIR_FLAG_NO_FILE_DIRS | DIR_FLAG_NO_FILE_INFO);
+
+ XBMC::XBMC_MD5 md5state;
+
+ if (excludes.size())
+ md5state.append(StringUtils::JoinString(excludes, "|"));
+
+ int64_t time = 0;
+ for (int i=0; i < items.Size(); ++i)
+ {
+ int64_t stat_time = 0;
+ struct __stat64 buffer;
+ if (XFILE::CFile::Stat(items[i]->GetPath(), &buffer) == 0)
+ {
+ // TODO: some filesystems may return the mtime/ctime inline, in which case this is
+ // unnecessarily expensive. Consider supporting Stat() in our directory cache?
+ stat_time = buffer.st_mtime ? buffer.st_mtime : buffer.st_ctime;
+ time += stat_time;
+ }
+
+ if (!stat_time)
+ return "";
+ }
+
+ if (time)
+ {
+ md5state.append((unsigned char *)&time, sizeof(time));
+ CStdString pathHash;
+ md5state.getDigest(pathHash);
+ return pathHash;
}
return "";
}
@@ -185,19 +185,35 @@ namespace VIDEO
Performs a stat() on the directory, and uses modified time to create a "fast"
hash of the folder. If no modified time is available, the create time is used,
and if neither are available, an empty hash is returned.
+ In case exclude from scan expressions are present, the string array will be appended
+ to the md5 hash to ensure we're doing a re-scan whenever the user modifies those.
\param directory folder to hash
- \return the hash of the folder of the form "fast<datetime>"
+ \param excludes string array of exclude expressions
+ \return the md5 hash of the folder"
*/
- CStdString GetFastHash(const CStdString &directory) const;
+ CStdString GetFastHash(const CStdString &directory, const CStdStringArray &excludes) const;
+
+ /*! \brief Retrieve a "fast" hash of the given directory recursively (if available)
+ Performs a stat() on the directory, and uses modified time to create a "fast"
+ hash of each folder. If no modified time is available, the create time is used,
+ and if neither are available, an empty hash is returned.
+ In case exclude from scan expressions are present, the string array will be appended
+ to the md5 hash to ensure we're doing a re-scan whenever the user modifies those.
+ \param directory folder to hash (recursively)
+ \param excludes string array of exclude expressions
+ \return the md5 hash of the folder
+ */
+ CStdString GetRecursiveFastHash(const CStdString &directory, const CStdStringArray &excludes) const;
/*! \brief Decide whether a folder listing could use the "fast" hash
Fast hashing can be done whenever the folder contains no scannable subfolders, as the
fast hash technique uses modified time to determine when folder content changes, which
is generally not propogated up the directory tree.
\param items the directory listing
+ \param excludes string array of exclude expressions
\return true if this directory listing can be fast hashed, false otherwise
*/
- bool CanFastHash(const CFileItemList &items) const;
+ bool CanFastHash(const CFileItemList &items, const CStdStringArray &excludes) const;
/*! \brief Process a series folder, filling in episode details and adding them to the database.
TODO: Ideally we would return INFO_HAVE_ALREADY if we don't have to update any episodes
@@ -212,7 +228,7 @@ namespace VIDEO
*/
INFO_RET OnProcessSeriesFolder(EPISODELIST& files, const ADDON::ScraperPtr &scraper, bool useLocal, const CVideoInfoTag& showInfo, CGUIDialogProgress* pDlgProgress = NULL);
- void EnumerateSeriesFolder(CFileItem* item, EPISODELIST& episodeList);
+ bool EnumerateSeriesFolder(CFileItem* item, EPISODELIST& episodeList);
bool EnumerateEpisodeItem(const CFileItem *item, EPISODELIST& episodeList);
bool ProcessItemByVideoInfoTag(const CFileItem *item, EPISODELIST &episodeList);

0 comments on commit 6afb499

Please sign in to comment.