Implemented serialization of pipeline caching for Vulkan compute. Fixes

#677.

scopehal/ComputePipeline.cpp

@@ -56,10 +56,8 @@ ComputePipeline::~ComputePipeline()
 
 void ComputePipeline::DeferredInit()
 {
-	double start = GetTime();
-
 	//Look up the pipeline cache to see if we have a binary etc to use
-	auto cache = g_pipelineCacheMgr->Lookup(m_shaderPath);
+	auto cache = g_pipelineCacheMgr->Lookup(BaseName(m_shaderPath));
 
 	//Load the shader module
 	auto srcvec = ReadDataFileUint32(m_shaderPath);
@@ -105,7 +103,4 @@ void ComputePipeline::DeferredInit()
 	vk::DescriptorSetAllocateInfo dsinfo(**m_descriptorPool, **m_descriptorSetLayout);
 	m_descriptorSet = make_unique<vk::raii::DescriptorSet>(
 		std::move(vk::raii::DescriptorSets(*g_vkComputeDevice, dsinfo).front()));
-
-	auto dt = GetTime() - start;
-	LogVerbose("Created compute pipeline for %s in %.3f ms\n", m_shaderPath.c_str(), dt * 1000);
 }

scopehal/PipelineCacheManager.cpp

@@ -102,7 +102,7 @@ void PipelineCacheManager::FindPath()
 	m_cacheRootDir = ExpandPath("~/.cache/glscopeclient") + "/";
 #endif
 
-	LogNotice("Cache root directory is %s\n", m_cacheRootDir.c_str());
+	LogTrace("Cache root directory is %s\n", m_cacheRootDir.c_str());
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -121,7 +121,7 @@ void PipelineCacheManager::Clear()
 /**
 	@brief Look up a blob which may or may not be in the cache
  */
-shared_ptr< vector<uint32_t> > PipelineCacheManager::LookupRaw(const string& key)
+shared_ptr< vector<uint8_t> > PipelineCacheManager::LookupRaw(const string& key)
 {
 	lock_guard<mutex> lock(m_mutex);
 	if(m_rawDataCache.find(key) != m_rawDataCache.end())
@@ -137,12 +137,12 @@ shared_ptr< vector<uint32_t> > PipelineCacheManager::LookupRaw(const string& key
 /**
 	@brief Store a raw blob to the cache
  */
-void PipelineCacheManager::StoreRaw(const string& key, shared_ptr< vector<uint32_t> > value)
+void PipelineCacheManager::StoreRaw(const string& key, shared_ptr< vector<uint8_t> > value)
 {
 	lock_guard<mutex> lock(m_mutex);
 	m_rawDataCache[key] = value;
 
-	LogTrace("Store raw: %s (%zu words)\n", key.c_str(), value->size());
+	LogTrace("Store raw: %s (%zu bytes)\n", key.c_str(), value->size());
 }
 
 /**
@@ -185,14 +185,23 @@ void PipelineCacheManager::LoadFromDisk()
 	PipelineCacheFileHeader header;
 	int vkfft_expected = VkFFTGetVersion();
 
+	string rawPrefix = "raw_";
+	string shaderPrefix = "pipeline_";
+	string shaderSuffix = ".bin";
+
 	//Load raw binary blobs (mostly for vkFFT)
-	auto prefix = m_cacheRootDir + "shader_raw_";
+	auto prefix = m_cacheRootDir + "shader_";
 	auto files = Glob(prefix + "*", false);
 	for(auto f : files)
 	{
 		//Extract the key from the file name
 		auto key = f.substr(prefix.length());
-		key = key.substr(0, key.length() - 4);
+		key = key.substr(0, key.length() - shaderSuffix.length());
+		bool typeIsRaw = (key.find(rawPrefix) == 0);
+		if(typeIsRaw)
+			key = key.substr(rawPrefix.length());
+		else
+			key = key.substr(shaderPrefix.length());
 
 		LogTrace("Loading cache object %s (from %s)\n", key.c_str(), f.c_str());
 		LogIndenter li2;
@@ -225,8 +234,8 @@ void PipelineCacheManager::LoadFromDisk()
 		}
 
 		//All good. Read the file content
-		auto p = make_shared< vector<uint32_t> >();
-		p->resize(header.len / 4);
+		auto p = make_shared< vector<uint8_t> >();
+		p->resize(header.len);
 		if(header.len != fread(&((*p)[0]), 1, header.len, fp))
 		{
 			LogWarning("Read cache content failed (%s)\n", f.c_str());
@@ -236,14 +245,22 @@ void PipelineCacheManager::LoadFromDisk()
 		fclose(fp);
 
 		//Verify the CRC
-		if(header.crc != CRC32((uint8_t*)&((*p)[0]), 0, header.len-1))
+		if(header.crc != CRC32(*p))
 		{
 			LogWarning("Rejecting cache file (%s) due to bad CRC\n", f.c_str());
 			continue;
 		}
 
 		//Done, add to cache if we get this far
-		m_rawDataCache[key] = p;
+		if(typeIsRaw)
+			m_rawDataCache[key] = p;
+		else
+		{
+			vector<uint8_t>& vec = *p;
+			vk::PipelineCacheCreateInfo info({}, vec.size(), &vec[0]);
+			auto ret = make_shared<vk::raii::PipelineCache>(*g_vkComputeDevice, info);
+			m_vkCache[key] = ret;
+		}
 	}
 }
 
@@ -254,6 +271,9 @@ void PipelineCacheManager::SaveToDisk()
 {
 	lock_guard<mutex> lock(m_mutex);
 
+	LogTrace("Saving cache\n");
+	LogIndenter li;
+
 	PipelineCacheFileHeader header;
 	memcpy(header.cache_uuid, g_vkComputeDeviceUuid, 16);
 	header.driver_ver = g_vkComputeDeviceDriverVer;
@@ -265,11 +285,12 @@ void PipelineCacheManager::SaveToDisk()
 		auto key = it.first;
 		auto& vec = *it.second;
 		auto fname = m_cacheRootDir + "shader_raw_" + key + ".bin";
+		LogTrace("Saving shader %s (%zu bytes)\n", fname.c_str(), vec.size());
 		FILE* fp = fopen(fname.c_str(), "wb");
 
 		//Write the cache header
-		header.len = vec.size() * sizeof(uint32_t);
-		header.crc = CRC32((uint8_t*)&vec[0], 0, header.len-1);
+		header.len = vec.size();
+		header.crc = CRC32(vec);
 		if(1 != fwrite(&header, sizeof(header), 1, fp))
 		{
 			LogWarning("Write cache header failed (%s)\n", fname.c_str());
@@ -284,5 +305,33 @@ void PipelineCacheManager::SaveToDisk()
 		fclose(fp);
 	}
 
-	//TODO: Save Vulkan shader cache
+	//Save Vulkan shader cache
+	for(auto it : m_vkCache)
+	{
+		auto key = it.first;
+		auto pcache = it.second;
+		auto fname = m_cacheRootDir + "shader_pipeline_" + key + ".bin";
+
+		//Extract the raw shader content
+		auto vec = pcache->getData();
+		LogTrace("Saving shader %s (%zu bytes)\n", fname.c_str(), vec.size());
+
+		FILE* fp = fopen(fname.c_str(), "wb");
+
+		//Write the cache header
+		header.len = vec.size();
+		header.crc = CRC32(vec);
+		if(1 != fwrite(&header, sizeof(header), 1, fp))
+		{
+			LogWarning("Write cache header failed (%s)\n", fname.c_str());
+			fclose(fp);
+			continue;
+		}
+
+		//Write the data
+		if(header.len != fwrite(&vec[0], 1, header.len, fp))
+			LogWarning("Write cache data failed (%s)\n", fname.c_str());
+
+		fclose(fp);
+	}
 }

scopehal/PipelineCacheManager.h

@@ -53,16 +53,16 @@ struct PipelineCacheFileHeader
 	The cache is stored on disk under the .cache/glscopeclient directory on Linux, or FIXME on Windows.
 
 	Raw data: $cachedir/shader_raw_[key].bin
-	Compute shader data: $cachedir/shader_compute_[key].bin
+	Vulkan shader data: $cachedir/shader_pipeline_[key].bin
  */
 class PipelineCacheManager
 {
 public:
 	PipelineCacheManager();
 	~PipelineCacheManager();
 
-	std::shared_ptr< std::vector<uint32_t> > LookupRaw(const std::string& key);
-	void StoreRaw(const std::string& key, std::shared_ptr< std::vector<uint32_t> > value);
+	std::shared_ptr< std::vector<uint8_t> > LookupRaw(const std::string& key);
+	void StoreRaw(const std::string& key, std::shared_ptr< std::vector<uint8_t> > value);
 
 	std::shared_ptr<vk::raii::PipelineCache> Lookup(const std::string& key);
 
@@ -80,7 +80,7 @@ class PipelineCacheManager
 	std::map<std::string, std::shared_ptr<vk::raii::PipelineCache> > m_vkCache;
 
 	///@brief The actual cache data store
-	std::map<std::string, std::shared_ptr<std::vector<uint32_t> > > m_rawDataCache;
+	std::map<std::string, std::shared_ptr<std::vector<uint8_t> > > m_rawDataCache;
 
 	///@brief Root directory of the cache
 	std::string m_cacheRootDir;

scopehal/VulkanFFTPlan.cpp

@@ -127,8 +127,8 @@ VulkanFFTPlan::VulkanFFTPlan(size_t npoints, size_t nouts, VulkanFFTPlanDirectio
 	//Add to cache if it wasn't there already
 	if(cacheBlob == nullptr)
 	{
-		auto vec = make_shared<vector<uint32_t> >();
-		vec->resize(m_app.applicationStringSize / sizeof(uint32_t));
+		auto vec = make_shared<vector<uint8_t> >();
+		vec->resize(m_app.applicationStringSize);
 		memcpy(&(*vec)[0], m_app.saveApplicationString, m_app.applicationStringSize);
 		g_pipelineCacheMgr->StoreRaw(cacheKey, vec);
 	}

scopehal/scopehal.cpp

@@ -889,7 +889,7 @@ void CreateDirectory(const string& path)
 /**
 	@brief Calculates a CRC32 checksum using the standard Ethernet polynomial
  */
-uint32_t CRC32(uint8_t* bytes, size_t start, size_t end)
+uint32_t CRC32(const uint8_t* bytes, size_t start, size_t end)
 {
 	uint32_t poly = 0xedb88320;
 
@@ -911,3 +911,8 @@ uint32_t CRC32(uint8_t* bytes, size_t start, size_t end)
 				((crc & 0x00ff0000) >> 8) |
 				 (crc >> 24) );
 }
+
+uint32_t CRC32(const vector<uint8_t>& bytes)
+{
+	return CRC32(&bytes[0], 0, bytes.size()-1);
+}

scopehal/scopehal.h

@@ -213,6 +213,7 @@ void CreateDirectory(const std::string& path);
 #endif
 
 //Checksum helpers
-uint32_t CRC32(uint8_t* bytes, size_t start, size_t end);
+uint32_t CRC32(const uint8_t* bytes, size_t start, size_t end);
+uint32_t CRC32(const std::vector<uint8_t>& bytes);
 
 #endif