Make Videos storage directory definitions global

This reworks the initial scan behavior for the Video library, making the
use of the Videos storage group consistent with those for recordings.
The hostnames specified for those groups are now ignored in effect,
allowing slave backends to use directories defined on the master
backend.

This is accomplished by adding a new method to supply a list of
available backends to query.  When the list of search paths are added in
later, the Myth URIs are extracted from the list, and collected in a
unique list of directories, in case the same directory is defined for
multiple backends.  All resultent directories are recombined with all
available backends, and added into the list.  Scan behavior for local
directories is left unchanged.  The failed list is replaced by an
offline list, which reports as a popup of inaccessible hosts with
existing video content referenced to them.

There is currently no mechanism to remove content lost to a failed
backend, besides setting up a replacement backend of the same name.
There is a flaw in the scan when performed by the master backend, where
it will not query remote backends for content lists. The current storage
group code will only write to directories defined locally for that
backend, meaning the various artwork storage groups still need to be
defined on each backend for that to work properly. I've held off
performing a schema update to move all of these definitions over to the
master backend until that artwork issue is resolved.

mythtv/libs/libmythmetadata/dirscan.cpp

@@ -141,6 +141,13 @@ namespace
             ok = true;
         }
         else
+            // FIXME: In the case of the master backend running the scan through
+            // the SCAN_VIDEOS command, the master backend will attempt to
+            // connect back to itself in order to scan slave backends, resulting
+            // in an error.  While this is interpreted as the slave backend
+            // being offline, and does not result in the deletion of content on
+            // a slave backend, it does mean the backend scanner will never be
+            // able to update content existing anywhere but the master backend.
             ok = RemoteGetFileList(host, start_path, &list, "Videos");
 
         if (!ok || (!list.isEmpty() && list.at(0).startsWith("SLAVE UNREACHABLE")))

mythtv/libs/libmythmetadata/metadatafactory.cpp

@@ -6,6 +6,7 @@
 #include "videoutils.h"
 #include "mythlogging.h"
 #include "compat.h"
+#include "remoteutil.h"
 
 // Needed to perform a lookup
 #include "metadatadownload.h"
@@ -226,12 +227,29 @@ bool MetadataFactory::VideoGrabbersFunctional()
 }
 
 void MetadataFactory::VideoScan()
+{
+    if (IsRunning())
+        return;
+
+    QStringList hosts;
+    if (!RemoteGetActiveBackends(&hosts))
+    {
+        LOG(VB_GENERAL, LOG_WARNING, "Could not retrieve list of "
+                            "available backends.");
+        return;
+    }
+
+    VideoScan(hosts);
+}
+
+void MetadataFactory::VideoScan(QStringList hosts)
 {
     if (IsRunning())
         return;
 
     m_scanning = true;
 
+    m_videoscanner->SetHosts(hosts);
     m_videoscanner->SetDirs(GetVideoDirs());
     m_videoscanner->start();
 }

mythtv/libs/libmythmetadata/metadatafactory.h

@@ -92,6 +92,7 @@ class META_PUBLIC MetadataFactory : public QObject
     MetadataLookupList SynchronousLookup(MetadataLookup *lookup);
 
     void VideoScan();
+    void VideoScan(QStringList hosts);
 
     bool IsRunning() { return m_lookupthread->isRunning() ||
                               m_imagedownload->isRunning() ||

mythtv/libs/libmythmetadata/videoscan.cpp

@@ -88,6 +88,34 @@ VideoScannerThread::~VideoScannerThread()
     delete m_dbmetadata;
 }
 
+void VideoScannerThread::SetDirs(QStringList dirs)
+{
+    QStringList sgdirs;
+
+    QStringList::iterator iter, iter2;
+    for (iter = dirs.begin(); iter != dirs.end(); ++iter)
+    {
+        if (iter->startsWith("myth://"))
+        {
+            QUrl sgurl = *iter;
+            iter = dirs.erase(iter);
+
+            if (!sgdirs.contains(sgurl.path()))
+                sgdirs.append(sgurl.path());
+        }
+    }
+
+    for (iter = m_liveSGHosts.begin(); iter != m_liveSGHosts.end(); ++iter)
+    {
+        for (iter2 = sgdirs.begin(); iter2 != sgdirs.end(); ++iter2)
+        {
+            dirs.append(gCoreContext->GenMythURL(*iter, 0, *iter2, "Videos"));
+        }
+    }
+
+    m_directories = dirs;
+}
+
 void VideoScannerThread::run()
 {
     RunProlog();
@@ -108,7 +136,6 @@ void VideoScannerThread::run()
 
     uint counter = 0;
     FileCheckList fs_files;
-    failedSGHosts.clear();
 
     if (m_HasGUI)
         SendProgressEvent(counter, (uint)m_directories.size(),
@@ -124,7 +151,7 @@ void VideoScannerThread::run()
                 QString host = sgurl.host();
                 QString path = sgurl.path();
 
-                failedSGHosts.append(host);
+                m_liveSGHosts.removeAll(host);
 
                 LOG(VB_GENERAL, LOG_ERR,
                     QString("Failed to scan :%1:").arg(*iter));
@@ -213,17 +240,21 @@ void VideoScannerThread::verifyFiles(FileCheckList &files,
                 // cannot reach, mark it as for removal later.
                 remove.push_back(std::make_pair((*p)->GetID(), lname));
             }
-            else if (!failedSGHosts.contains(lhost))
+            else if (m_liveSGHosts.contains(lhost))
             {
                 LOG(VB_GENERAL, LOG_INFO,
                     QString("Removing file SG(%1) :%2:")
                         .arg(lhost).arg(lname));
                 remove.push_back(std::make_pair((*p)->GetID(), lname));
             }
             else
+            {
                 LOG(VB_GENERAL, LOG_WARNING,
                     QString("SG(%1) not available. Not removing file :%2:")
                         .arg(lhost).arg(lname));
+                if (!m_offlineSGHosts.contains(lhost))
+                    m_offlineSGHosts.append(lhost);
+            }
         }
         if (m_HasGUI)
             SendProgressEvent(++counter);
@@ -317,6 +348,12 @@ bool VideoScannerThread::buildFileList(const QString &directory,
                                        const QStringList &imageExtensions,
                                        FileCheckList &filelist)
 {
+    // TODO: FileCheckList is a std::map, keyed off the filename. In the event
+    // multiple backends have access to shared storage, the potential exists
+    // for files to be scanned onto the wrong host. Add in some logic to prefer
+    // the backend with the content stored in a storage group determined to be
+    // local.
+
     LOG(VB_GENERAL,LOG_INFO, QString("buildFileList directory = %1")
                                  .arg(directory));
     FileAssociations::ext_ignore_list ext_list;
@@ -376,6 +413,14 @@ void VideoScanner::doScan(const QStringList &dirs)
         m_scanThread->SetProgressDialog(progressDlg);
     }
 
+    QStringList hosts;
+    if (!RemoteGetActiveBackends(&hosts))
+    {
+        LOG(VB_GENERAL, LOG_WARNING, "Could not retrieve list of "
+                            "available backends.");
+        hosts.clear();
+    }
+    m_scanThread->SetHosts(hosts);
     m_scanThread->SetDirs(dirs);
     m_scanThread->start();
 }
@@ -387,7 +432,7 @@ void VideoScanner::doScanAll()
 
 void VideoScanner::finishedScan()
 {
-    QStringList failedHosts = m_scanThread->GetFailedSGHosts();
+    QStringList failedHosts = m_scanThread->GetOfflineSGHosts();
     if (failedHosts.size() > 0)
     {
         QString msg = tr("Failed to Scan SG Video Hosts") + ":\n\n";

mythtv/libs/libmythmetadata/videoscan.h

@@ -63,9 +63,10 @@ class META_PUBLIC VideoScannerThread : public MThread
     ~VideoScannerThread();
 
     void run();
-    void SetDirs(const QStringList &dirs) { m_directories = dirs; };
+    void SetDirs(QStringList dirs);
+    void SetHosts(const QStringList &hosts) { m_liveSGHosts = hosts; };
     void SetProgressDialog(MythUIProgressDialog *dialog) { m_dialog = dialog; };
-    QStringList GetFailedSGHosts(void) { return failedSGHosts; };
+    QStringList GetOfflineSGHosts(void) { return m_offlineSGHosts; };
     bool getDataChanged() { return m_DBDataChanged; };
 
     void ResetCounts() { m_addList.clear(); m_movList.clear(); m_delList.clear(); };
@@ -99,7 +100,8 @@ class META_PUBLIC VideoScannerThread : public MThread
     bool m_KeepAll;
     bool m_HasGUI;
     QStringList m_directories;
-    QStringList failedSGHosts;
+    QStringList m_liveSGHosts;
+    QStringList m_offlineSGHosts;
 
     VideoMetadataListManager *m_dbmetadata;
     MythUIProgressDialog *m_dialog;