From 1693a18ec1e668f2b6e3e703f60a7da85ed40c7c Mon Sep 17 00:00:00 2001 From: Raymond Wagner Date: Sat, 11 Feb 2012 15:12:40 -0500 Subject: [PATCH] Make Videos storage directory definitions global This reworks the initial scan behavior for the Video library, making the use of the Videos storage group consistent with those for recordings. The hostnames specified for those groups are now ignored in effect, allowing slave backends to use directories defined on the master backend. This is accomplished by adding a new method to supply a list of available backends to query. When the list of search paths are added in later, the Myth URIs are extracted from the list, and collected in a unique list of directories, in case the same directory is defined for multiple backends. All resultent directories are recombined with all available backends, and added into the list. Scan behavior for local directories is left unchanged. The failed list is replaced by an offline list, which reports as a popup of inaccessible hosts with existing video content referenced to them. There is currently no mechanism to remove content lost to a failed backend, besides setting up a replacement backend of the same name. There is a flaw in the scan when performed by the master backend, where it will not query remote backends for content lists. The current storage group code will only write to directories defined locally for that backend, meaning the various artwork storage groups still need to be defined on each backend for that to work properly. I've held off performing a schema update to move all of these definitions over to the master backend until that artwork issue is resolved. --- mythtv/libs/libmythmetadata/dirscan.cpp | 7 +++ .../libs/libmythmetadata/metadatafactory.cpp | 18 +++++++ mythtv/libs/libmythmetadata/metadatafactory.h | 1 + mythtv/libs/libmythmetadata/videoscan.cpp | 53 +++++++++++++++++-- mythtv/libs/libmythmetadata/videoscan.h | 8 +-- 5 files changed, 80 insertions(+), 7 deletions(-) diff --git a/mythtv/libs/libmythmetadata/dirscan.cpp b/mythtv/libs/libmythmetadata/dirscan.cpp index 621e0da6e45..e16d36ed59d 100644 --- a/mythtv/libs/libmythmetadata/dirscan.cpp +++ b/mythtv/libs/libmythmetadata/dirscan.cpp @@ -141,6 +141,13 @@ namespace ok = true; } else + // FIXME: In the case of the master backend running the scan through + // the SCAN_VIDEOS command, the master backend will attempt to + // connect back to itself in order to scan slave backends, resulting + // in an error. While this is interpreted as the slave backend + // being offline, and does not result in the deletion of content on + // a slave backend, it does mean the backend scanner will never be + // able to update content existing anywhere but the master backend. ok = RemoteGetFileList(host, start_path, &list, "Videos"); if (!ok || (!list.isEmpty() && list.at(0).startsWith("SLAVE UNREACHABLE"))) diff --git a/mythtv/libs/libmythmetadata/metadatafactory.cpp b/mythtv/libs/libmythmetadata/metadatafactory.cpp index 44cf473e695..27603c5867e 100644 --- a/mythtv/libs/libmythmetadata/metadatafactory.cpp +++ b/mythtv/libs/libmythmetadata/metadatafactory.cpp @@ -6,6 +6,7 @@ #include "videoutils.h" #include "mythlogging.h" #include "compat.h" +#include "remoteutil.h" // Needed to perform a lookup #include "metadatadownload.h" @@ -226,12 +227,29 @@ bool MetadataFactory::VideoGrabbersFunctional() } void MetadataFactory::VideoScan() +{ + if (IsRunning()) + return; + + QStringList hosts; + if (!RemoteGetActiveBackends(&hosts)) + { + LOG(VB_GENERAL, LOG_WARNING, "Could not retrieve list of " + "available backends."); + return; + } + + VideoScan(hosts); +} + +void MetadataFactory::VideoScan(QStringList hosts) { if (IsRunning()) return; m_scanning = true; + m_videoscanner->SetHosts(hosts); m_videoscanner->SetDirs(GetVideoDirs()); m_videoscanner->start(); } diff --git a/mythtv/libs/libmythmetadata/metadatafactory.h b/mythtv/libs/libmythmetadata/metadatafactory.h index 00cc4d9665e..6c2599d2364 100644 --- a/mythtv/libs/libmythmetadata/metadatafactory.h +++ b/mythtv/libs/libmythmetadata/metadatafactory.h @@ -92,6 +92,7 @@ class META_PUBLIC MetadataFactory : public QObject MetadataLookupList SynchronousLookup(MetadataLookup *lookup); void VideoScan(); + void VideoScan(QStringList hosts); bool IsRunning() { return m_lookupthread->isRunning() || m_imagedownload->isRunning() || diff --git a/mythtv/libs/libmythmetadata/videoscan.cpp b/mythtv/libs/libmythmetadata/videoscan.cpp index c427171391e..7d62ba64874 100644 --- a/mythtv/libs/libmythmetadata/videoscan.cpp +++ b/mythtv/libs/libmythmetadata/videoscan.cpp @@ -88,6 +88,34 @@ VideoScannerThread::~VideoScannerThread() delete m_dbmetadata; } +void VideoScannerThread::SetDirs(QStringList dirs) +{ + QStringList sgdirs; + + QStringList::iterator iter, iter2; + for (iter = dirs.begin(); iter != dirs.end(); ++iter) + { + if (iter->startsWith("myth://")) + { + QUrl sgurl = *iter; + iter = dirs.erase(iter); + + if (!sgdirs.contains(sgurl.path())) + sgdirs.append(sgurl.path()); + } + } + + for (iter = m_liveSGHosts.begin(); iter != m_liveSGHosts.end(); ++iter) + { + for (iter2 = sgdirs.begin(); iter2 != sgdirs.end(); ++iter2) + { + dirs.append(gCoreContext->GenMythURL(*iter, 0, *iter2, "Videos")); + } + } + + m_directories = dirs; +} + void VideoScannerThread::run() { RunProlog(); @@ -108,7 +136,6 @@ void VideoScannerThread::run() uint counter = 0; FileCheckList fs_files; - failedSGHosts.clear(); if (m_HasGUI) SendProgressEvent(counter, (uint)m_directories.size(), @@ -124,7 +151,7 @@ void VideoScannerThread::run() QString host = sgurl.host(); QString path = sgurl.path(); - failedSGHosts.append(host); + m_liveSGHosts.removeAll(host); LOG(VB_GENERAL, LOG_ERR, QString("Failed to scan :%1:").arg(*iter)); @@ -213,7 +240,7 @@ void VideoScannerThread::verifyFiles(FileCheckList &files, // cannot reach, mark it as for removal later. remove.push_back(std::make_pair((*p)->GetID(), lname)); } - else if (!failedSGHosts.contains(lhost)) + else if (m_liveSGHosts.contains(lhost)) { LOG(VB_GENERAL, LOG_INFO, QString("Removing file SG(%1) :%2:") @@ -221,9 +248,13 @@ void VideoScannerThread::verifyFiles(FileCheckList &files, remove.push_back(std::make_pair((*p)->GetID(), lname)); } else + { LOG(VB_GENERAL, LOG_WARNING, QString("SG(%1) not available. Not removing file :%2:") .arg(lhost).arg(lname)); + if (!m_offlineSGHosts.contains(lhost)) + m_offlineSGHosts.append(lhost); + } } if (m_HasGUI) SendProgressEvent(++counter); @@ -317,6 +348,12 @@ bool VideoScannerThread::buildFileList(const QString &directory, const QStringList &imageExtensions, FileCheckList &filelist) { + // TODO: FileCheckList is a std::map, keyed off the filename. In the event + // multiple backends have access to shared storage, the potential exists + // for files to be scanned onto the wrong host. Add in some logic to prefer + // the backend with the content stored in a storage group determined to be + // local. + LOG(VB_GENERAL,LOG_INFO, QString("buildFileList directory = %1") .arg(directory)); FileAssociations::ext_ignore_list ext_list; @@ -376,6 +413,14 @@ void VideoScanner::doScan(const QStringList &dirs) m_scanThread->SetProgressDialog(progressDlg); } + QStringList hosts; + if (!RemoteGetActiveBackends(&hosts)) + { + LOG(VB_GENERAL, LOG_WARNING, "Could not retrieve list of " + "available backends."); + hosts.clear(); + } + m_scanThread->SetHosts(hosts); m_scanThread->SetDirs(dirs); m_scanThread->start(); } @@ -387,7 +432,7 @@ void VideoScanner::doScanAll() void VideoScanner::finishedScan() { - QStringList failedHosts = m_scanThread->GetFailedSGHosts(); + QStringList failedHosts = m_scanThread->GetOfflineSGHosts(); if (failedHosts.size() > 0) { QString msg = tr("Failed to Scan SG Video Hosts") + ":\n\n"; diff --git a/mythtv/libs/libmythmetadata/videoscan.h b/mythtv/libs/libmythmetadata/videoscan.h index 1bb5211bf8b..7e693f25f66 100644 --- a/mythtv/libs/libmythmetadata/videoscan.h +++ b/mythtv/libs/libmythmetadata/videoscan.h @@ -63,9 +63,10 @@ class META_PUBLIC VideoScannerThread : public MThread ~VideoScannerThread(); void run(); - void SetDirs(const QStringList &dirs) { m_directories = dirs; }; + void SetDirs(QStringList dirs); + void SetHosts(const QStringList &hosts) { m_liveSGHosts = hosts; }; void SetProgressDialog(MythUIProgressDialog *dialog) { m_dialog = dialog; }; - QStringList GetFailedSGHosts(void) { return failedSGHosts; }; + QStringList GetOfflineSGHosts(void) { return m_offlineSGHosts; }; bool getDataChanged() { return m_DBDataChanged; }; void ResetCounts() { m_addList.clear(); m_movList.clear(); m_delList.clear(); }; @@ -99,7 +100,8 @@ class META_PUBLIC VideoScannerThread : public MThread bool m_KeepAll; bool m_HasGUI; QStringList m_directories; - QStringList failedSGHosts; + QStringList m_liveSGHosts; + QStringList m_offlineSGHosts; VideoMetadataListManager *m_dbmetadata; MythUIProgressDialog *m_dialog;