diff --git a/Source/Core/Common/MemoryUtil.cpp b/Source/Core/Common/MemoryUtil.cpp index 9455212f017a..510b258b09e0 100644 --- a/Source/Core/Common/MemoryUtil.cpp +++ b/Source/Core/Common/MemoryUtil.cpp @@ -19,6 +19,12 @@ #else #include #include +#include +#ifdef __APPLE__ +#include +#else +#include +#endif #endif // Valgrind doesn't support MAP_32BIT. @@ -241,3 +247,26 @@ std::string MemUsage() return ""; #endif } + + +size_t MemPhysical() +{ +#ifdef _WIN32 + MEMORYSTATUSEX memInfo; + memInfo.dwLength = sizeof(MEMORYSTATUSEX); + GlobalMemoryStatusEx(&memInfo); + return memInfo.ullTotalPhys; +#elif defined(__APPLE__) + int mib[2]; + size_t physical_memory; + mib[0] = CTL_HW; + mib[1] = HW_MEMSIZE; + size_t length = sizeof(size_t); + sysctl(mib, 2, &physical_memory, &length, NULL, 0); + return physical_memory; +#else + struct sysinfo memInfo; + sysinfo (&memInfo); + return (size_t)memInfo.totalram * memInfo.mem_unit; +#endif +} diff --git a/Source/Core/Common/MemoryUtil.h b/Source/Core/Common/MemoryUtil.h index f8ba087c0f9e..e986069d247f 100644 --- a/Source/Core/Common/MemoryUtil.h +++ b/Source/Core/Common/MemoryUtil.h @@ -16,6 +16,7 @@ void ReadProtectMemory(void* ptr, size_t size); void WriteProtectMemory(void* ptr, size_t size, bool executable = false); void UnWriteProtectMemory(void* ptr, size_t size, bool allowExecute = false); std::string MemUsage(); +size_t MemPhysical(); void GuardMemoryMake(void* ptr, size_t size); void GuardMemoryUnmake(void* ptr, size_t size); diff --git a/Source/Core/DolphinWX/VideoConfigDiag.cpp b/Source/Core/DolphinWX/VideoConfigDiag.cpp index c703761554c7..943abe3cb290 100644 --- a/Source/Core/DolphinWX/VideoConfigDiag.cpp +++ b/Source/Core/DolphinWX/VideoConfigDiag.cpp @@ -131,6 +131,7 @@ static wxString xfb_virtual_desc = _("Emulate XFBs using GPU texture objects.\nF static wxString xfb_real_desc = _("Emulate XFBs accurately.\nSlows down emulation a lot and prohibits high-resolution rendering but is necessary to emulate a number of games properly.\n\nIf unsure, check virtual XFB emulation instead."); static wxString dump_textures_desc = _("Dump decoded game textures to User/Dump/Textures//.\n\nIf unsure, leave this unchecked."); static wxString load_hires_textures_desc = _("Load custom textures from User/Load/Textures//.\n\nIf unsure, leave this unchecked."); +static wxString cache_hires_textures_desc = _("Cache custom textures to system RAM on startup.\nThis can require exponentially more RAM but fixes possible stuttering.\n\nIf unsure, leave this unchecked."); static wxString dump_efb_desc = _("Dump the contents of EFB copies to User/Dump/Textures/.\n\nIf unsure, leave this unchecked."); #if !defined WIN32 && defined HAVE_LIBAV static wxString use_ffv1_desc = _("Encode frame dumps using the FFV1 codec.\n\nIf unsure, leave this unchecked."); @@ -556,6 +557,8 @@ VideoConfigDiag::VideoConfigDiag(wxWindow* parent, const std::string &title, con szr_utility->Add(CreateCheckBox(page_advanced, _("Dump Textures"), dump_textures_desc, vconfig.bDumpTextures)); szr_utility->Add(CreateCheckBox(page_advanced, _("Load Custom Textures"), load_hires_textures_desc, vconfig.bHiresTextures)); + cache_hires_textures = CreateCheckBox(page_advanced, _("Prefetch Custom Textures"), cache_hires_textures_desc, vconfig.bCacheHiresTextures); + szr_utility->Add(cache_hires_textures); szr_utility->Add(CreateCheckBox(page_advanced, _("Dump EFB Target"), dump_efb_desc, vconfig.bDumpEFBTarget)); szr_utility->Add(CreateCheckBox(page_advanced, _("Free Look"), free_look_desc, vconfig.bFreeLook)); #if !defined WIN32 && defined HAVE_LIBAV diff --git a/Source/Core/DolphinWX/VideoConfigDiag.h b/Source/Core/DolphinWX/VideoConfigDiag.h index 829239ed66b7..e7939216b770 100644 --- a/Source/Core/DolphinWX/VideoConfigDiag.h +++ b/Source/Core/DolphinWX/VideoConfigDiag.h @@ -199,6 +199,9 @@ class VideoConfigDiag : public wxDialog virtual_xfb->Enable(vconfig.bUseXFB); real_xfb->Enable(vconfig.bUseXFB); + // custom textures + cache_hires_textures->Enable(vconfig.bHiresTextures); + // Repopulating the post-processing shaders can't be done from an event if (choice_ppshader && choice_ppshader->IsEmpty()) PopulatePostProcessingShaders(); @@ -262,6 +265,8 @@ class VideoConfigDiag : public wxDialog SettingRadioButton* virtual_xfb; SettingRadioButton* real_xfb; + SettingCheckBox* cache_hires_textures; + wxCheckBox* progressive_scan_checkbox; wxChoice* choice_ppshader; diff --git a/Source/Core/VideoCommon/HiresTextures.cpp b/Source/Core/VideoCommon/HiresTextures.cpp index 0e2989fa2444..ac764fcd4ffd 100644 --- a/Source/Core/VideoCommon/HiresTextures.cpp +++ b/Source/Core/VideoCommon/HiresTextures.cpp @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -13,7 +15,11 @@ #include "Common/CommonPaths.h" #include "Common/FileSearch.h" #include "Common/FileUtil.h" +#include "Common/Flag.h" +#include "Common/MemoryUtil.h" #include "Common/StringUtil.h" +#include "Common/Thread.h" +#include "Common/Timer.h" #include "Core/ConfigManager.h" @@ -22,18 +28,59 @@ #include "VideoCommon/VideoConfig.h" static std::unordered_map s_textureMap; +static std::unordered_map> s_textureCache; +static std::mutex s_textureCacheMutex; +static std::mutex s_textureCacheAquireMutex; // for high priority access +static Common::Flag s_textureCacheAbortLoading; static bool s_check_native_format; static bool s_check_new_format; +static std::thread s_prefetcher; + static const std::string s_format_prefix = "tex1_"; -void HiresTexture::Init(const std::string& gameCode) +void HiresTexture::Init() { - s_textureMap.clear(); s_check_native_format = false; s_check_new_format = false; + Update(); +} + +void HiresTexture::Shutdown() +{ + if (s_prefetcher.joinable()) + { + s_textureCacheAbortLoading.Set(); + s_prefetcher.join(); + } + + s_textureMap.clear(); + s_textureCache.clear(); +} + +void HiresTexture::Update() +{ + if (s_prefetcher.joinable()) + { + s_textureCacheAbortLoading.Set(); + s_prefetcher.join(); + } + + if (!g_ActiveConfig.bHiresTextures) + { + s_textureMap.clear(); + s_textureCache.clear(); + return; + } + + if (!g_ActiveConfig.bCacheHiresTextures) + { + s_textureCache.clear(); + } + CFileSearch::XStringVector Directories; + const std::string& gameCode = SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID; std::string szDir = StringFromFormat("%s%s", File::GetUserPath(D_HIRESTEXTURES_IDX).c_str(), gameCode.c_str()); Directories.push_back(szDir); @@ -94,6 +141,81 @@ void HiresTexture::Init(const std::string& gameCode) s_check_new_format = true; } } + + if (g_ActiveConfig.bCacheHiresTextures) + { + // remove cached but deleted textures + auto iter = s_textureCache.begin(); + while (iter != s_textureCache.end()) + { + if (s_textureMap.find(iter->first) == s_textureMap.end()) + { + iter = s_textureCache.erase(iter); + } + else + { + iter++; + } + } + + s_textureCacheAbortLoading.Clear(); + s_prefetcher = std::thread(Prefetch); + } +} + +void HiresTexture::Prefetch() +{ + Common::SetCurrentThreadName("Prefetcher"); + + size_t size_sum = 0; + size_t max_mem = MemPhysical() / 2; + u32 starttime = Common::Timer::GetTimeMs(); + for (const auto& entry : s_textureMap) + { + const std::string& base_filename = entry.first; + + if (base_filename.find("_mip") == std::string::npos) + { + { + // try to get this mutex first, so the video thread is allow to get the real mutex faster + std::unique_lock lk(s_textureCacheAquireMutex); + } + std::unique_lock lk(s_textureCacheMutex); + + auto iter = s_textureCache.find(base_filename); + if (iter == s_textureCache.end()) + { + // unlock while loading a texture. This may result in a race condition where we'll load a texture twice, + // but it reduces the stuttering a lot. Notice: The loading library _must_ be thread safe now. + // But bad luck, SOIL isn't, so TODO: remove SOIL usage here and use libpng directly + // Also TODO: remove s_textureCacheAquireMutex afterwards. It won't be needed as the main mutex will be locked rarely + //lk.unlock(); + std::shared_ptr ptr(Load(base_filename, 0, 0)); + //lk.lock(); + + iter = s_textureCache.insert(iter, std::make_pair(base_filename, ptr)); + } + for (const Level& l : iter->second->m_levels) + { + size_sum += l.data_size; + } + } + + if (s_textureCacheAbortLoading.IsSet()) + { + return; + } + + if (size_sum > max_mem) + { + g_Config.bCacheHiresTextures = false; + + OSD::AddMessage(StringFromFormat("Custom Textures prefetching after %.1f MB aborted, not enough RAM available", size_sum / (1024.0 * 1024.0)), 10000); + return; + } + } + u32 stoptime = Common::Timer::GetTimeMs(); + OSD::AddMessage(StringFromFormat("Custom Textures loaded, %.1f MB in %.1f s", size_sum / (1024.0 * 1024.0), (stoptime - starttime) / 1000.0), 10000); } std::string HiresTexture::GenBaseName(const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, int format, bool has_mipmaps, bool dump) @@ -238,10 +360,31 @@ std::string HiresTexture::GenBaseName(const u8* texture, size_t texture_size, co return name; } -HiresTexture* HiresTexture::Search(const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, int format, bool has_mipmaps) +std::shared_ptr HiresTexture::Search(const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, int format, bool has_mipmaps) { std::string base_filename = GenBaseName(texture, texture_size, tlut, tlut_size, width, height, format, has_mipmaps); + std::lock_guard lk2(s_textureCacheAquireMutex); + std::lock_guard lk(s_textureCacheMutex); + + auto iter = s_textureCache.find(base_filename); + if (iter != s_textureCache.end()) + { + return iter->second; + } + + std::shared_ptr ptr(Load(base_filename, width, height)); + + if (ptr && g_ActiveConfig.bCacheHiresTextures) + { + s_textureCache[base_filename] = ptr; + } + + return ptr; +} + +HiresTexture* HiresTexture::Load(const std::string& base_filename, u32 width, u32 height) +{ HiresTexture* ret = nullptr; for (int level = 0;; level++) { @@ -275,7 +418,7 @@ HiresTexture* HiresTexture::Search(const u8* texture, size_t texture_size, const if (l.width * height != l.height * width) ERROR_LOG(VIDEO, "Invalid custom texture size %dx%d for texture %s. The aspect differs from the native size %dx%d.", l.width, l.height, filename.c_str(), width, height); - if (l.width % width || l.height % height) + if (width && height && (l.width % width || l.height % height)) WARN_LOG(VIDEO, "Invalid custom texture size %dx%d for texture %s. Please use an integer upscaling factor based on the native size %dx%d.", l.width, l.height, filename.c_str(), width, height); width = l.width; diff --git a/Source/Core/VideoCommon/HiresTextures.h b/Source/Core/VideoCommon/HiresTextures.h index 6ffb7e462601..65a033de7d29 100644 --- a/Source/Core/VideoCommon/HiresTextures.h +++ b/Source/Core/VideoCommon/HiresTextures.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include #include "VideoCommon/TextureDecoder.h" @@ -12,9 +13,11 @@ class HiresTexture { public: - static void Init(const std::string& gameCode); + static void Init(); + static void Update(); + static void Shutdown(); - static HiresTexture* Search( + static std::shared_ptr Search( const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, @@ -40,6 +43,9 @@ class HiresTexture std::vector m_levels; private: + static HiresTexture* Load(const std::string& base_filename, u32 width, u32 height); + static void Prefetch(); + HiresTexture() {} }; diff --git a/Source/Core/VideoCommon/TextureCacheBase.cpp b/Source/Core/VideoCommon/TextureCacheBase.cpp index bce936661e5f..ac882ec3df46 100644 --- a/Source/Core/VideoCommon/TextureCacheBase.cpp +++ b/Source/Core/VideoCommon/TextureCacheBase.cpp @@ -60,8 +60,7 @@ TextureCache::TextureCache() TexDecoder_SetTexFmtOverlayOptions(g_ActiveConfig.bTexFmtOverlayEnable, g_ActiveConfig.bTexFmtOverlayCenter); - if (g_ActiveConfig.bHiresTextures && !g_ActiveConfig.bDumpTextures) - HiresTexture::Init(SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID); + HiresTexture::Init(); SetHash64Function(); @@ -92,6 +91,7 @@ void TextureCache::Invalidate() TextureCache::~TextureCache() { + HiresTexture::Shutdown(); Invalidate(); FreeAlignedMemory(temp); temp = nullptr; @@ -101,6 +101,12 @@ void TextureCache::OnConfigChanged(VideoConfig& config) { if (g_texture_cache) { + if (config.bHiresTextures != backup_config.s_hires_textures || + config.bCacheHiresTextures != backup_config.s_cache_hires_textures) + { + HiresTexture::Update(); + } + // TODO: Invalidating texcache is really stupid in some of these cases if (config.iSafeTextureCache_ColorSamples != backup_config.s_colorsamples || config.bTexFmtOverlayEnable != backup_config.s_texfmt_overlay || @@ -110,9 +116,6 @@ void TextureCache::OnConfigChanged(VideoConfig& config) { g_texture_cache->Invalidate(); - if (g_ActiveConfig.bHiresTextures) - HiresTexture::Init(SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID); - TexDecoder_SetTexFmtOverlayOptions(g_ActiveConfig.bTexFmtOverlayEnable, g_ActiveConfig.bTexFmtOverlayCenter); invalidate_texture_cache_requested = false; @@ -130,6 +133,7 @@ void TextureCache::OnConfigChanged(VideoConfig& config) backup_config.s_texfmt_overlay = config.bTexFmtOverlayEnable; backup_config.s_texfmt_overlay_center = config.bTexFmtOverlayCenter; backup_config.s_hires_textures = config.bHiresTextures; + backup_config.s_cache_hires_textures = config.bCacheHiresTextures; backup_config.s_stereo_3d = config.iStereoMode > 0; backup_config.s_efb_mono_depth = config.bStereoEFBMonoDepth; } @@ -433,15 +437,15 @@ TextureCache::TCacheEntryBase* TextureCache::Load(const u32 stage) textures.erase(oldest_entry); } - std::unique_ptr hires_tex; + std::shared_ptr hires_tex; if (g_ActiveConfig.bHiresTextures) { - hires_tex.reset(HiresTexture::Search( + hires_tex = HiresTexture::Search( src_data, texture_size, &texMem[tlutaddr], palette_size, width, height, texformat, use_mipmaps - )); + ); if (hires_tex) { diff --git a/Source/Core/VideoCommon/TextureCacheBase.h b/Source/Core/VideoCommon/TextureCacheBase.h index 73182b41c7e0..7cc5d1e87587 100644 --- a/Source/Core/VideoCommon/TextureCacheBase.h +++ b/Source/Core/VideoCommon/TextureCacheBase.h @@ -153,6 +153,7 @@ class TextureCache bool s_texfmt_overlay; bool s_texfmt_overlay_center; bool s_hires_textures; + bool s_cache_hires_textures; bool s_copy_cache_enable; bool s_stereo_3d; bool s_efb_mono_depth; diff --git a/Source/Core/VideoCommon/VideoConfig.cpp b/Source/Core/VideoCommon/VideoConfig.cpp index 9bca0e86adc6..1a989878f558 100644 --- a/Source/Core/VideoCommon/VideoConfig.cpp +++ b/Source/Core/VideoCommon/VideoConfig.cpp @@ -71,6 +71,7 @@ void VideoConfig::Load(const std::string& ini_file) settings->Get("DumpTextures", &bDumpTextures, 0); settings->Get("HiresTextures", &bHiresTextures, 0); settings->Get("ConvertHiresTextures", &bConvertHiresTextures, 0); + settings->Get("CacheHiresTextures", &bCacheHiresTextures, 0); settings->Get("DumpEFBTarget", &bDumpEFBTarget, 0); settings->Get("FreeLook", &bFreeLook, 0); settings->Get("UseFFV1", &bUseFFV1, 0); @@ -151,6 +152,7 @@ void VideoConfig::GameIniLoad() CHECK_SETTING("Video_Settings", "SafeTextureCacheColorSamples", iSafeTextureCache_ColorSamples); CHECK_SETTING("Video_Settings", "HiresTextures", bHiresTextures); CHECK_SETTING("Video_Settings", "ConvertHiresTextures", bConvertHiresTextures); + CHECK_SETTING("Video_Settings", "CacheHiresTextures", bCacheHiresTextures); CHECK_SETTING("Video_Settings", "EnablePixelLighting", bEnablePixelLighting); CHECK_SETTING("Video_Settings", "FastDepthCalc", bFastDepthCalc); CHECK_SETTING("Video_Settings", "MSAA", iMultisampleMode); @@ -258,6 +260,7 @@ void VideoConfig::Save(const std::string& ini_file) settings->Set("DumpTextures", bDumpTextures); settings->Set("HiresTextures", bHiresTextures); settings->Set("ConvertHiresTextures", bConvertHiresTextures); + settings->Set("CacheHiresTextures", bCacheHiresTextures); settings->Set("DumpEFBTarget", bDumpEFBTarget); settings->Set("FreeLook", bFreeLook); settings->Set("UseFFV1", bUseFFV1); diff --git a/Source/Core/VideoCommon/VideoConfig.h b/Source/Core/VideoCommon/VideoConfig.h index 64eec52173c2..9635c90d19ed 100644 --- a/Source/Core/VideoCommon/VideoConfig.h +++ b/Source/Core/VideoCommon/VideoConfig.h @@ -104,6 +104,7 @@ struct VideoConfig final bool bDumpTextures; bool bHiresTextures; bool bConvertHiresTextures; + bool bCacheHiresTextures; bool bDumpEFBTarget; bool bUseFFV1; bool bFreeLook;