Merge pull request #5659 from shuffle2/gamelist-speedup-with-emustate

Gamelist speedup with emustate

Source/Core/Common/ChunkFile.h

@@ -296,125 +296,3 @@ class PointerWrap
     *ptr += size;
   }
 };
-
-// NOTE: this class is only used in DolphinWX/ISOFile.cpp for caching loaded
-// ISO data. It will be removed when DolphinWX is, so please don't use it.
-class CChunkFileReader
-{
-public:
-  // Load file template
-  template <class T>
-  static bool Load(const std::string& _rFilename, u32 _Revision, T& _class)
-  {
-    INFO_LOG(COMMON, "ChunkReader: Loading %s", _rFilename.c_str());
-
-    if (!File::Exists(_rFilename))
-      return false;
-
-    // Check file size
-    const u64 fileSize = File::GetSize(_rFilename);
-    static const u64 headerSize = sizeof(SChunkHeader);
-    if (fileSize < headerSize)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: File too small");
-      return false;
-    }
-
-    File::IOFile pFile(_rFilename, "rb");
-    if (!pFile)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Can't open file for reading");
-      return false;
-    }
-
-    // read the header
-    SChunkHeader header;
-    if (!pFile.ReadArray(&header, 1))
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Bad header size");
-      return false;
-    }
-
-    // Check revision
-    if (header.Revision != _Revision)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Wrong file revision, got %d expected %d", header.Revision,
-                _Revision);
-      return false;
-    }
-
-    // get size
-    const u32 sz = (u32)(fileSize - headerSize);
-    if (header.ExpectedSize != sz)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Bad file size, got %d expected %d", sz, header.ExpectedSize);
-      return false;
-    }
-
-    // read the state
-    std::vector<u8> buffer(sz);
-    if (!pFile.ReadArray(&buffer[0], sz))
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Error reading file");
-      return false;
-    }
-
-    u8* ptr = &buffer[0];
-    PointerWrap p(&ptr, PointerWrap::MODE_READ);
-    _class.DoState(p);
-
-    INFO_LOG(COMMON, "ChunkReader: Done loading %s", _rFilename.c_str());
-    return true;
-  }
-
-  // Save file template
-  template <class T>
-  static bool Save(const std::string& _rFilename, u32 _Revision, T& _class)
-  {
-    INFO_LOG(COMMON, "ChunkReader: Writing %s", _rFilename.c_str());
-    File::IOFile pFile(_rFilename, "wb");
-    if (!pFile)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Error opening file for write");
-      return false;
-    }
-
-    // Get data
-    u8* ptr = nullptr;
-    PointerWrap p(&ptr, PointerWrap::MODE_MEASURE);
-    _class.DoState(p);
-    size_t const sz = (size_t)ptr;
-    std::vector<u8> buffer(sz);
-    ptr = &buffer[0];
-    p.SetMode(PointerWrap::MODE_WRITE);
-    _class.DoState(p);
-
-    // Create header
-    SChunkHeader header;
-    header.Revision = _Revision;
-    header.ExpectedSize = (u32)sz;
-
-    // Write to file
-    if (!pFile.WriteArray(&header, 1))
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Failed writing header");
-      return false;
-    }
-
-    if (!pFile.WriteArray(&buffer[0], sz))
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Failed writing data");
-      return false;
-    }
-
-    INFO_LOG(COMMON, "ChunkReader: Done writing %s", _rFilename.c_str());
-    return true;
-  }
-
-private:
-  struct SChunkHeader
-  {
-    u32 Revision;
-    u32 ExpectedSize;
-  };
-};

Source/Core/Common/FileSearch.cpp

@@ -7,10 +7,19 @@
 
 #include "Common/CommonPaths.h"
 #include "Common/FileSearch.h"
+
+#ifdef _MSC_VER
+#include <experimental/filesystem>
+namespace fs = std::experimental::filesystem;
+#define HAS_STD_FILESYSTEM
+#else
 #include "Common/FileUtil.h"
+#endif
 
 namespace Common
 {
+#ifndef HAS_STD_FILESYSTEM
+
 static std::vector<std::string>
 FileSearchWithTest(const std::vector<std::string>& directories, bool recursive,
                    std::function<bool(const File::FSTEntry&)> callback)
@@ -36,10 +45,10 @@ FileSearchWithTest(const std::vector<std::string>& directories, bool recursive,
   return result;
 }
 
-std::vector<std::string> DoFileSearch(const std::vector<std::string>& exts,
-                                      const std::vector<std::string>& directories, bool recursive)
+std::vector<std::string> DoFileSearchNoSTL(const std::vector<std::string>& directories,
+                                           const std::vector<std::string>& exts, bool recursive)
 {
-  bool accept_all = std::find(exts.begin(), exts.end(), "") != exts.end();
+  bool accept_all = exts.empty();
   return FileSearchWithTest(directories, recursive, [&](const File::FSTEntry& entry) {
     if (accept_all)
       return true;
@@ -52,11 +61,71 @@ std::vector<std::string> DoFileSearch(const std::vector<std::string>& exts,
   });
 }
 
-// Result includes the passed directories themselves as well as their subdirectories.
-std::vector<std::string> FindSubdirectories(const std::vector<std::string>& directories,
-                                            bool recursive)
+std::vector<std::string> DoFileSearch(const std::vector<std::string>& directories,
+                                      const std::vector<std::string>& exts, bool recursive)
 {
-  return FileSearchWithTest(directories, true,
-                            [&](const File::FSTEntry& entry) { return entry.isDirectory; });
+  return DoFileSearchNoSTL(directories, exts, recursive);
 }
+
+#else
+
+std::vector<std::string> DoFileSearch(const std::vector<std::string>& directories,
+                                      const std::vector<std::string>& exts, bool recursive)
+{
+  bool accept_all = exts.empty();
+
+  std::vector<fs::path> native_exts;
+  for (const auto& ext : exts)
+    native_exts.push_back(ext);
+
+  // N.B. This avoids doing any copies
+  auto ext_matches = [&native_exts](const fs::path& path) {
+    const auto& native_path = path.native();
+    return std::any_of(native_exts.cbegin(), native_exts.cend(), [&native_path](const auto& ext) {
+      // TODO provide cross-platform compat for the comparison function, once more platforms
+      // support std::filesystem
+      return native_path.length() >= ext.native().length() &&
+             _wcsicmp(&native_path.c_str()[native_path.length() - ext.native().length()],
+                      ext.c_str()) == 0;
+    });
+  };
+
+  std::vector<std::string> result;
+  auto add_filtered = [&](const fs::directory_entry& entry) {
+    auto& path = entry.path();
+    if (accept_all || (ext_matches(path) && !fs::is_directory(path)))
+      result.emplace_back(path.u8string());
+  };
+  for (const auto& directory : directories)
+  {
+    if (recursive)
+    {
+      // TODO use fs::directory_options::follow_directory_symlink ?
+      for (auto& entry : fs::recursive_directory_iterator(fs::path(directory.c_str())))
+        add_filtered(entry);
+    }
+    else
+    {
+      for (auto& entry : fs::directory_iterator(fs::path(directory.c_str())))
+        add_filtered(entry);
+    }
+  }
+
+  // Remove duplicates (occurring because caller gave e.g. duplicate or overlapping directories -
+  // not because std::filesystem returns duplicates). Also note that this pathname-based uniqueness
+  // isn't as thorough as std::filesystem::equivalent.
+  std::sort(result.begin(), result.end());
+  result.erase(std::unique(result.begin(), result.end()), result.end());
+
+  // Dolphin expects to be able to use "/" (DIR_SEP) everywhere. std::filesystem uses the OS
+  // separator.
+  if (fs::path::preferred_separator != DIR_SEP_CHR)
+    for (auto& path : result)
+      std::replace(path.begin(), path.end(), '\\', DIR_SEP_CHR);
+
+  return result;
+}
+
+#endif
+
 }  // namespace Common

Source/Core/Common/FileSearch.h

@@ -9,9 +9,9 @@
 
 namespace Common
 {
-std::vector<std::string> DoFileSearch(const std::vector<std::string>& exts,
-                                      const std::vector<std::string>& directories,
+// Callers can pass empty "exts" to indicate they want all files + directories in results
+// Otherwise, only files matching the extensions are returned
+std::vector<std::string> DoFileSearch(const std::vector<std::string>& directories,
+                                      const std::vector<std::string>& exts = {},
                                       bool recursive = false);
-std::vector<std::string> FindSubdirectories(const std::vector<std::string>& directories,
-                                            bool recursive);
 }  // namespace Common

Source/Core/Core/HW/GCMemcard/GCMemcardDirectory.cpp

@@ -148,7 +148,7 @@ GCMemcardDirectory::GCMemcardDirectory(const std::string& directory, int slot, u
     hdr_file.ReadBytes(&m_hdr, BLOCK_SIZE);
   }
 
-  std::vector<std::string> filenames = Common::DoFileSearch({".gci"}, {m_save_directory});
+  std::vector<std::string> filenames = Common::DoFileSearch({m_save_directory}, {".gci"});
 
   if (filenames.size() > 112)
   {

Source/Core/DolphinQt2/Settings.cpp

@@ -365,7 +365,7 @@ QVector<QString> Settings::GetProfiles(const InputConfig* config) const
   const std::string path = GetProfilesDir().toStdString() + config->GetProfileName();
   QVector<QString> vec;
 
-  for (const auto& file : Common::DoFileSearch({".ini"}, {path}))
+  for (const auto& file : Common::DoFileSearch({path}, {".ini"}))
   {
     std::string basename;
     SplitPath(file, nullptr, &basename, nullptr);

Source/Core/DolphinQt2/Settings/InterfacePane.cpp

@@ -59,9 +59,8 @@ void InterfacePane::CreateUI()
   combobox_layout->addRow(tr("&Theme:"), m_combobox_theme);
 
   // List avalable themes
-  auto file_search_results = Common::DoFileSearch(
-      {""}, {File::GetUserPath(D_THEMES_IDX), File::GetSysDirectory() + THEMES_DIR},
-      /*recursive*/ false);
+  auto file_search_results =
+      Common::DoFileSearch({File::GetUserPath(D_THEMES_IDX), File::GetSysDirectory() + THEMES_DIR});
   for (const std::string& filename : file_search_results)
   {
     std::string name, ext;

Source/Core/DolphinWX/Config/ConfigMain.cpp

@@ -24,21 +24,21 @@
 #include "DolphinWX/GameListCtrl.h"
 #include "DolphinWX/WxUtils.h"
 
-// Sent by child panes to signify that the game list should
-// be updated when this modal dialog closes.
 wxDEFINE_EVENT(wxDOLPHIN_CFG_REFRESH_LIST, wxCommandEvent);
+wxDEFINE_EVENT(wxDOLPHIN_CFG_RESCAN_LIST, wxCommandEvent);
 
 CConfigMain::CConfigMain(wxWindow* parent, wxWindowID id, const wxString& title,
                          const wxPoint& position, const wxSize& size, long style)
     : wxDialog(parent, id, title, position, size, style)
 {
-  // Control refreshing of the ISOs list
-  m_refresh_game_list_on_close = false;
+  // Control refreshing of the GameListCtrl
+  m_event_on_close = wxEVT_NULL;
 
   Bind(wxEVT_CLOSE_WINDOW, &CConfigMain::OnClose, this);
   Bind(wxEVT_BUTTON, &CConfigMain::OnCloseButton, this, wxID_CLOSE);
   Bind(wxEVT_SHOW, &CConfigMain::OnShow, this);
   Bind(wxDOLPHIN_CFG_REFRESH_LIST, &CConfigMain::OnSetRefreshGameListOnClose, this);
+  Bind(wxDOLPHIN_CFG_RESCAN_LIST, &CConfigMain::OnSetRescanGameListOnClose, this);
 
   wxDialog::SetExtraStyle(GetExtraStyle() & ~wxWS_EX_BLOCK_EVENTS);
 
@@ -115,8 +115,8 @@ void CConfigMain::OnClose(wxCloseEvent& WXUNUSED(event))
 
   SConfig::GetInstance().SaveSettings();
 
-  if (m_refresh_game_list_on_close)
-    AddPendingEvent(wxCommandEvent{DOLPHIN_EVT_RELOAD_GAMELIST});
+  if (m_event_on_close != wxEVT_NULL)
+    AddPendingEvent(wxCommandEvent{m_event_on_close});
 }
 
 void CConfigMain::OnShow(wxShowEvent& event)
@@ -132,5 +132,12 @@ void CConfigMain::OnCloseButton(wxCommandEvent& WXUNUSED(event))
 
 void CConfigMain::OnSetRefreshGameListOnClose(wxCommandEvent& WXUNUSED(event))
 {
-  m_refresh_game_list_on_close = true;
+  // Don't override a rescan
+  if (m_event_on_close == wxEVT_NULL)
+    m_event_on_close = DOLPHIN_EVT_REFRESH_GAMELIST;
+}
+
+void CConfigMain::OnSetRescanGameListOnClose(wxCommandEvent& WXUNUSED(event))
+{
+  m_event_on_close = DOLPHIN_EVT_RESCAN_GAMELIST;
 }

Source/Core/DolphinWX/Config/ConfigMain.h

@@ -10,7 +10,10 @@
 class wxNotebook;
 class wxPanel;
 
+// Fast refresh - can be fulfilled from cache
 wxDECLARE_EVENT(wxDOLPHIN_CFG_REFRESH_LIST, wxCommandEvent);
+// Rescan and refresh - modifies cache
+wxDECLARE_EVENT(wxDOLPHIN_CFG_RESCAN_LIST, wxCommandEvent);
 
 class CConfigMain : public wxDialog
 {
@@ -41,8 +44,8 @@ class CConfigMain : public wxDialog
   void OnCloseButton(wxCommandEvent& event);
   void OnShow(wxShowEvent& event);
   void OnSetRefreshGameListOnClose(wxCommandEvent& event);
+  void OnSetRescanGameListOnClose(wxCommandEvent& event);
 
   wxNotebook* Notebook;
-
-  bool m_refresh_game_list_on_close;
+  wxEventType m_event_on_close;
 };

Source/Core/DolphinWX/Config/InterfaceConfigPane.cpp

@@ -196,9 +196,8 @@ void InterfaceConfigPane::LoadGUIValues()
 
 void InterfaceConfigPane::LoadThemes()
 {
-  auto sv = Common::DoFileSearch(
-      {""}, {File::GetUserPath(D_THEMES_IDX), File::GetSysDirectory() + THEMES_DIR},
-      /*recursive*/ false);
+  auto sv =
+      Common::DoFileSearch({File::GetUserPath(D_THEMES_IDX), File::GetSysDirectory() + THEMES_DIR});
   for (const std::string& filename : sv)
   {
     std::string name, ext;

Source/Core/DolphinWX/Config/PathConfigPane.cpp

@@ -163,7 +163,7 @@ void PathConfigPane::OnRecursiveISOCheckBoxChanged(wxCommandEvent& event)
 {
   SConfig::GetInstance().m_RecursiveISOFolder = m_recursive_iso_paths_checkbox->IsChecked();
 
-  AddPendingEvent(wxCommandEvent(wxDOLPHIN_CFG_REFRESH_LIST));
+  AddPendingEvent(wxCommandEvent(wxDOLPHIN_CFG_RESCAN_LIST));
 }
 
 void PathConfigPane::OnAddISOPath(wxCommandEvent& event)
@@ -179,7 +179,7 @@ void PathConfigPane::OnAddISOPath(wxCommandEvent& event)
     }
     else
     {
-      AddPendingEvent(wxCommandEvent(wxDOLPHIN_CFG_REFRESH_LIST));
+      AddPendingEvent(wxCommandEvent(wxDOLPHIN_CFG_RESCAN_LIST));
       m_iso_paths_listbox->Append(dialog.GetPath());
     }
   }
@@ -189,7 +189,7 @@ void PathConfigPane::OnAddISOPath(wxCommandEvent& event)
 
 void PathConfigPane::OnRemoveISOPath(wxCommandEvent& event)
 {
-  AddPendingEvent(wxCommandEvent(wxDOLPHIN_CFG_REFRESH_LIST));
+  AddPendingEvent(wxCommandEvent(wxDOLPHIN_CFG_RESCAN_LIST));
   m_iso_paths_listbox->Delete(m_iso_paths_listbox->GetSelection());
 
 // This seems to not be activated on Windows when it should be. wxw bug?