diff --git a/level_zero/core/source/compiler_interface/default_cache_config.cpp b/level_zero/core/source/compiler_interface/default_cache_config.cpp index dd2b601900f2e..7a8a74f11b806 100644 --- a/level_zero/core/source/compiler_interface/default_cache_config.cpp +++ b/level_zero/core/source/compiler_interface/default_cache_config.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2022 Intel Corporation + * Copyright (C) 2020-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -9,6 +9,7 @@ #include "shared/source/compiler_interface/default_cache_config.h" +#include "shared/source/helpers/constants.h" #include "shared/source/utilities/debug_settings_reader.h" #include "level_zero/core/source/compiler_interface/l0_reg_path.h" @@ -24,6 +25,7 @@ CompilerCacheConfig getDefaultCompilerCacheConfig() { std::unique_ptr settingsReader(NEO::SettingsReader::createOsReader(false, keyName)); ret.cacheDir = settingsReader->getSetting(settingsReader->appSpecificLocation(keyName), static_cast(L0_CACHE_LOCATION)); + ret.cacheSize = MemoryConstants::gigaByte; ret.cacheFileExtension = ".l0_cache"; return ret; diff --git a/opencl/source/compiler_interface/default_cache_config.cpp b/opencl/source/compiler_interface/default_cache_config.cpp index 9a77ef3a47d71..e1d24caf329a0 100644 --- a/opencl/source/compiler_interface/default_cache_config.cpp +++ b/opencl/source/compiler_interface/default_cache_config.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2022 Intel Corporation + * Copyright (C) 2020-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -7,6 +7,7 @@ #include "shared/source/compiler_interface/default_cache_config.h" +#include "shared/source/helpers/constants.h" #include "shared/source/utilities/debug_settings_reader.h" #include "opencl/source/os_interface/ocl_reg_path.h" @@ -25,6 +26,7 @@ CompilerCacheConfig getDefaultCompilerCacheConfig() { std::unique_ptr settingsReader(SettingsReader::createOsReader(false, keyName)); ret.cacheDir = settingsReader->getSetting(settingsReader->appSpecificLocation(keyName), static_cast(CL_CACHE_LOCATION)); + ret.cacheSize = MemoryConstants::gigaByte; ret.cacheFileExtension = ".cl_cache"; return ret; diff --git a/opencl/test/unit_test/compiler_interface/default_cl_cache_config_tests.cpp b/opencl/test/unit_test/compiler_interface/default_cl_cache_config_tests.cpp index 0274cf10ec018..0c39d0182aa4a 100644 --- a/opencl/test/unit_test/compiler_interface/default_cl_cache_config_tests.cpp +++ b/opencl/test/unit_test/compiler_interface/default_cl_cache_config_tests.cpp @@ -14,19 +14,3 @@ TEST(CompilerCache, GivenDefaultClCacheConfigThenValuesAreProperlyPopulated) { EXPECT_STREQ(".cl_cache", cacheConfig.cacheFileExtension.c_str()); EXPECT_TRUE(cacheConfig.enabled); } - -TEST(CompilerCacheTests, GivenExistingConfigWhenLoadingFromCacheThenBinaryIsLoaded) { - NEO::CompilerCache cache(NEO::getDefaultCompilerCacheConfig()); - static const char *hash = "SOME_HASH"; - std::unique_ptr data(new char[32]); - for (size_t i = 0; i < 32; i++) - data.get()[i] = static_cast(i); - - bool ret = cache.cacheBinary(hash, static_cast(data.get()), 32); - EXPECT_TRUE(ret); - - size_t size; - auto loadedBin = cache.loadCachedBinary(hash, size); - EXPECT_NE(nullptr, loadedBin); - EXPECT_NE(0U, size); -} \ No newline at end of file diff --git a/shared/offline_compiler/source/CMakeLists.txt b/shared/offline_compiler/source/CMakeLists.txt index 2c45c7cd173c5..9f54f43bd5d0c 100644 --- a/shared/offline_compiler/source/CMakeLists.txt +++ b/shared/offline_compiler/source/CMakeLists.txt @@ -124,6 +124,7 @@ endif() if(WIN32) list(APPEND CLOC_LIB_SRCS_LIB + ${NEO_SHARED_DIRECTORY}/compiler_interface/windows/compiler_cache_windows.cpp ${NEO_SHARED_DIRECTORY}/dll/windows/options_windows.cpp ${NEO_SHARED_DIRECTORY}/os_interface/windows/os_inc.h ${NEO_SHARED_DIRECTORY}/os_interface/windows/os_library_win.cpp @@ -132,6 +133,7 @@ if(WIN32) ) else() list(APPEND CLOC_LIB_SRCS_LIB + ${NEO_SHARED_DIRECTORY}/compiler_interface/linux/compiler_cache_linux.cpp ${NEO_SHARED_DIRECTORY}/dll/linux/options_linux.cpp ${NEO_SHARED_DIRECTORY}/os_interface/linux/os_inc.h ${NEO_SHARED_DIRECTORY}/os_interface/linux/os_library_linux.cpp diff --git a/shared/source/compiler_interface/CMakeLists.txt b/shared/source/compiler_interface/CMakeLists.txt index 370dcc9dd4828..173502e7a6a73 100644 --- a/shared/source/compiler_interface/CMakeLists.txt +++ b/shared/source/compiler_interface/CMakeLists.txt @@ -26,4 +26,15 @@ set(NEO_CORE_COMPILER_INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/tokenized_string.h ) +if(WIN32) + list(APPEND NEO_CORE_COMPILER_INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/windows/compiler_cache_windows.cpp + ) +else() + list(APPEND NEO_CORE_COMPILER_INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/linux/compiler_cache_linux.cpp + ) +endif() + set_property(GLOBAL PROPERTY NEO_CORE_COMPILER_INTERFACE ${NEO_CORE_COMPILER_INTERFACE}) +add_subdirectories() diff --git a/shared/source/compiler_interface/compiler_cache.cpp b/shared/source/compiler_interface/compiler_cache.cpp index 92e93b7a7591a..a697097388053 100644 --- a/shared/source/compiler_interface/compiler_cache.cpp +++ b/shared/source/compiler_interface/compiler_cache.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2022 Intel Corporation + * Copyright (C) 2019-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ namespace NEO { std::mutex CompilerCache::cacheAccessMtx; + const std::string CompilerCache::getCachedFileName(const HardwareInfo &hwInfo, const ArrayRef input, const ArrayRef options, const ArrayRef internalOptions) { Hash hash; @@ -109,20 +110,4 @@ const std::string CompilerCache::getCachedFileName(const HardwareInfo &hwInfo, c CompilerCache::CompilerCache(const CompilerCacheConfig &cacheConfig) : config(cacheConfig){}; -bool CompilerCache::cacheBinary(const std::string kernelFileHash, const char *pBinary, uint32_t binarySize) { - if (pBinary == nullptr || binarySize == 0) { - return false; - } - std::string filePath = config.cacheDir + PATH_SEPARATOR + kernelFileHash + config.cacheFileExtension; - std::lock_guard lock(cacheAccessMtx); - return 0 != writeDataToFile(filePath.c_str(), pBinary, binarySize); -} - -std::unique_ptr CompilerCache::loadCachedBinary(const std::string kernelFileHash, size_t &cachedBinarySize) { - std::string filePath = config.cacheDir + PATH_SEPARATOR + kernelFileHash + config.cacheFileExtension; - - std::lock_guard lock(cacheAccessMtx); - return loadDataFromFile(filePath.c_str(), cachedBinarySize); -} - } // namespace NEO diff --git a/shared/source/compiler_interface/compiler_cache.h b/shared/source/compiler_interface/compiler_cache.h index 0289ef9e0cdb3..968da24bdc0e3 100644 --- a/shared/source/compiler_interface/compiler_cache.h +++ b/shared/source/compiler_interface/compiler_cache.h @@ -13,6 +13,7 @@ #include #include #include +#include namespace NEO { struct HardwareInfo; @@ -21,6 +22,7 @@ struct CompilerCacheConfig { bool enabled = true; std::string cacheFileExtension; std::string cacheDir; + size_t cacheSize = 0; }; class CompilerCache { @@ -36,10 +38,15 @@ class CompilerCache { const std::string getCachedFileName(const HardwareInfo &hwInfo, ArrayRef input, ArrayRef options, ArrayRef internalOptions); - MOCKABLE_VIRTUAL bool cacheBinary(const std::string kernelFileHash, const char *pBinary, uint32_t binarySize); - MOCKABLE_VIRTUAL std::unique_ptr loadCachedBinary(const std::string kernelFileHash, size_t &cachedBinarySize); + MOCKABLE_VIRTUAL bool cacheBinary(const std::string &kernelFileHash, const char *pBinary, size_t binarySize); + MOCKABLE_VIRTUAL std::unique_ptr loadCachedBinary(const std::string &kernelFileHash, size_t &cachedBinarySize); protected: + MOCKABLE_VIRTUAL bool evictCache(); + MOCKABLE_VIRTUAL bool renameTempFileBinaryToProperName(const std::string &oldName, const std::string &kernelFileHash); + MOCKABLE_VIRTUAL bool createUniqueTempFileAndWriteData(char *tmpFilePathTemplate, const char *pBinary, size_t binarySize); + MOCKABLE_VIRTUAL void lockConfigFileAndReadSize(const std::string &configFilePath, int &fd, size_t &directorySize); + static std::mutex cacheAccessMtx; CompilerCacheConfig config; }; diff --git a/shared/source/compiler_interface/linux/compiler_cache_linux.cpp b/shared/source/compiler_interface/linux/compiler_cache_linux.cpp new file mode 100644 index 0000000000000..031097a0fd8d6 --- /dev/null +++ b/shared/source/compiler_interface/linux/compiler_cache_linux.cpp @@ -0,0 +1,281 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/compiler_interface/compiler_cache.h" +#include "shared/source/debug_settings/debug_settings_manager.h" +#include "shared/source/helpers/file_io.h" +#include "shared/source/helpers/string.h" +#include "shared/source/os_interface/linux/sys_calls.h" +#include "shared/source/utilities/io_functions.h" + +#include "os_inc.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace NEO { +std::string makePath(const std::string &lhs, const std::string &rhs) { + if (lhs.size() == 0) { + return rhs; + } + + if (rhs.size() == 0) { + return lhs; + } + + if (*lhs.rbegin() == PATH_SEPARATOR) { + return lhs + rhs; + } + + return lhs + PATH_SEPARATOR + rhs; +} + +int filterFunction(const struct dirent *file) { + std::string_view fileName = file->d_name; + if (fileName.find(".cl_cache") != fileName.npos || fileName.find(".l0_cache") != fileName.npos) { + return 1; + } + + return 0; +} + +struct ElementsStruct { + std::string path; + struct stat statEl; +}; + +bool compareByLastAccessTime(const ElementsStruct &a, ElementsStruct &b) { + return a.statEl.st_atime < b.statEl.st_atime; +} + +bool CompilerCache::evictCache() { + struct dirent **files = 0; + + int filesCount = NEO::SysCalls::scandir(config.cacheDir.c_str(), &files, filterFunction, NULL); + + if (filesCount == -1) { + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "PID %d [Cache failure]: Scandir failed! errno: %d\n", NEO::SysCalls::getProcessId(), errno); + return false; + } + + std::vector vec; + vec.reserve(static_cast(filesCount)); + for (int i = 0; i < filesCount; ++i) { + ElementsStruct fileElement = {}; + fileElement.path = makePath(config.cacheDir, files[i]->d_name); + if (NEO::SysCalls::stat(fileElement.path.c_str(), &fileElement.statEl) == 0) { + vec.push_back(std::move(fileElement)); + } + } + + for (int i = 0; i < filesCount; ++i) { + free(files[i]); + } + free(files); + + std::sort(vec.begin(), vec.end(), compareByLastAccessTime); + + size_t evictionLimit = config.cacheSize / 3; + + size_t evictionSizeCount = 0; + for (size_t i = 0; i < vec.size(); ++i) { + NEO::SysCalls::unlink(vec[i].path); + evictionSizeCount += vec[i].statEl.st_size; + + if (evictionSizeCount > evictionLimit) { + return true; + } + } + + return true; +} + +bool CompilerCache::createUniqueTempFileAndWriteData(char *tmpFilePathTemplate, const char *pBinary, size_t binarySize) { + int fd = NEO::SysCalls::mkstemp(tmpFilePathTemplate); + if (fd == -1) { + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "PID %d [Cache failure]: Creating temporary file failed! errno: %d\n", NEO::SysCalls::getProcessId(), errno); + return false; + } + if (NEO::SysCalls::pwrite(fd, pBinary, binarySize, 0) == -1) { + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "PID %d [Cache failure]: Writing to temporary file failed! errno: %d\n", NEO::SysCalls::getProcessId(), errno); + NEO::SysCalls::close(fd); + NEO::SysCalls::unlink(tmpFilePathTemplate); + return false; + } + + return NEO::SysCalls::close(fd) == 0; +} + +bool CompilerCache::renameTempFileBinaryToProperName(const std::string &oldName, const std::string &kernelFileHash) { + int err = NEO::SysCalls::rename(oldName.c_str(), kernelFileHash.c_str()); + + if (err < 0) { + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "PID %d [Cache failure]: Rename temp file failed! errno: %d\n", NEO::SysCalls::getProcessId(), errno); + NEO::SysCalls::unlink(oldName); + return false; + } + + return true; +} + +void unlockFileAndClose(int fd) { + int lockErr = NEO::SysCalls::flock(fd, LOCK_UN); + + if (lockErr < 0) { + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "PID %d [Cache failure]: unlock file failed! errno: %d\n", NEO::SysCalls::getProcessId(), errno); + } + + NEO::SysCalls::close(fd); +} + +void CompilerCache::lockConfigFileAndReadSize(const std::string &configFilePath, int &fd, size_t &directorySize) { + bool countDirectorySize = false; + errno = 0; + fd = NEO::SysCalls::open(configFilePath.c_str(), O_RDWR); + + if (fd < 0) { + if (errno == ENOENT) { + fd = NEO::SysCalls::openWithMode(configFilePath.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH); + if (fd < 0) { + fd = NEO::SysCalls::open(configFilePath.c_str(), O_RDWR); + } else { + countDirectorySize = true; + } + } else { + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "PID %d [Cache failure]: Open config file failed! errno: %d\n", NEO::SysCalls::getProcessId(), errno); + return; + } + } + + int lockErr = NEO::SysCalls::flock(fd, LOCK_EX); + + if (lockErr < 0) { + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "PID %d [Cache failure]: Lock config file failed! errno: %d\n", NEO::SysCalls::getProcessId(), errno); + NEO::SysCalls::close(fd); + fd = -1; + + return; + } + + if (countDirectorySize) { + struct dirent **files = {}; + + int filesCount = NEO::SysCalls::scandir(config.cacheDir.c_str(), &files, filterFunction, NULL); + + if (filesCount == -1) { + unlockFileAndClose(fd); + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "PID %d [Cache failure]: Scandir failed! errno: %d\n", NEO::SysCalls::getProcessId(), errno); + fd = -1; + return; + } + + std::vector vec; + vec.reserve(static_cast(filesCount)); + for (int i = 0; i < filesCount; ++i) { + std::string_view fileName = files[i]->d_name; + if (fileName.find(config.cacheFileExtension) != fileName.npos) { + ElementsStruct fileElement = {}; + fileElement.path = makePath(config.cacheDir, files[i]->d_name); + if (NEO::SysCalls::stat(fileElement.path.c_str(), &fileElement.statEl) == 0) { + vec.push_back(std::move(fileElement)); + } + } + } + + for (int i = 0; i < filesCount; ++i) { + free(files[i]); + } + free(files); + + for (auto &element : vec) { + directorySize += element.statEl.st_size; + } + + } else { + ssize_t readErr = NEO::SysCalls::pread(fd, &directorySize, sizeof(directorySize), 0); + + if (readErr < 0) { + directorySize = 0; + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "PID %d [Cache failure]: Read config failed! errno: %d\n", NEO::SysCalls::getProcessId(), errno); + unlockFileAndClose(fd); + fd = -1; + } + } +} + +bool CompilerCache::cacheBinary(const std::string &kernelFileHash, const char *pBinary, size_t binarySize) { + if (pBinary == nullptr || binarySize == 0) { + return false; + } + + std::unique_lock lock(cacheAccessMtx); + constexpr std::string_view configFileName = "config.file"; + + std::string configFilePath = makePath(config.cacheDir, configFileName.data()); + std::string filePath = makePath(config.cacheDir, kernelFileHash + config.cacheFileExtension); + + int fd = -1; + size_t directorySize = 0u; + + lockConfigFileAndReadSize(configFilePath, fd, directorySize); + + if (fd < 0) { + return false; + } + + struct stat statbuf = {}; + if (NEO::SysCalls::stat(filePath, &statbuf) == 0) { + unlockFileAndClose(fd); + return true; + } + + size_t maxSize = config.cacheSize; + + if (maxSize < directorySize + binarySize) { + if (!evictCache()) { + unlockFileAndClose(fd); + return false; + } + } + + std::string tmpFileName = "cl_cache.XXXXXX"; + + std::string tmpFilePath = makePath(config.cacheDir, tmpFileName); + + if (!createUniqueTempFileAndWriteData(tmpFilePath.data(), pBinary, binarySize)) { + unlockFileAndClose(fd); + return false; + } + + if (!renameTempFileBinaryToProperName(tmpFilePath, filePath)) { + unlockFileAndClose(fd); + return false; + } + + directorySize += binarySize; + + NEO::SysCalls::pwrite(fd, &directorySize, sizeof(directorySize), 0); + + unlockFileAndClose(fd); + + return true; +} + +std::unique_ptr CompilerCache::loadCachedBinary(const std::string &kernelFileHash, size_t &cachedBinarySize) { + std::string filePath = makePath(config.cacheDir, kernelFileHash + config.cacheFileExtension); + + return loadDataFromFile(filePath.c_str(), cachedBinarySize); +} +} // namespace NEO diff --git a/shared/source/compiler_interface/windows/compiler_cache_windows.cpp b/shared/source/compiler_interface/windows/compiler_cache_windows.cpp new file mode 100644 index 0000000000000..b220c46eb257b --- /dev/null +++ b/shared/source/compiler_interface/windows/compiler_cache_windows.cpp @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/compiler_interface/compiler_cache.h" +#include "shared/source/helpers/file_io.h" +#include "shared/source/utilities/io_functions.h" + +#include "os_inc.h" + +namespace NEO { +bool CompilerCache::evictCache() { + return true; +} + +void CompilerCache::lockConfigFileAndReadSize(const std::string &configFilePath, int &fd, size_t &directorySize) {} + +bool CompilerCache::createUniqueTempFileAndWriteData(char *tmpFilePath, const char *pBinary, size_t binarySize) { + return true; +} +bool CompilerCache::renameTempFileBinaryToProperName(const std::string &oldName, const std::string &kernelFileHash) { + return true; +} + +bool CompilerCache::cacheBinary(const std::string &kernelFileHash, const char *pBinary, size_t binarySize) { + if (pBinary == nullptr || binarySize == 0) { + return false; + } + std::string filePath = config.cacheDir + PATH_SEPARATOR + kernelFileHash + config.cacheFileExtension; + std::lock_guard lock(cacheAccessMtx); + return 0 != writeDataToFile(filePath.c_str(), pBinary, binarySize); +} + +std::unique_ptr CompilerCache::loadCachedBinary(const std::string &kernelFileHash, size_t &cachedBinarySize) { + std::string filePath = config.cacheDir + PATH_SEPARATOR + kernelFileHash + config.cacheFileExtension; + + std::lock_guard lock(cacheAccessMtx); + return loadDataFromFile(filePath.c_str(), cachedBinarySize); +} +} // namespace NEO \ No newline at end of file diff --git a/shared/source/os_interface/linux/sys_calls.h b/shared/source/os_interface/linux/sys_calls.h index 6b9141e69912c..9f8987452c617 100644 --- a/shared/source/os_interface/linux/sys_calls.h +++ b/shared/source/os_interface/linux/sys_calls.h @@ -8,6 +8,7 @@ #pragma once #include "shared/source/os_interface/sys_calls_common.h" +#include #include #include #include @@ -17,6 +18,7 @@ namespace NEO { namespace SysCalls { int close(int fileDescriptor); int open(const char *file, int flags); +int openWithMode(const char *file, int flags, int mode); void *dlopen(const char *filename, int flag); int ioctl(int fileDescriptor, unsigned long int request, void *arg); int getDevicePath(int deviceFd, char *buf, size_t &bufSize); @@ -33,6 +35,16 @@ ssize_t write(int fd, void *buf, size_t count); int fcntl(int fd, int cmd); int fcntl(int fd, int cmd, int arg); char *realpath(const char *path, char *buf); +int stat(const std::string &filePath, struct stat *statbuf); int pipe(int pipefd[2]); +int flock(int fd, int flag); +int mkstemp(char *filePath); +int rename(const char *currName, const char *dstName); +int scandir(const char *dirp, + struct dirent ***namelist, + int (*filter)(const struct dirent *), + int (*compar)(const struct dirent **, + const struct dirent **)); +int unlink(const std::string &pathname); } // namespace SysCalls } // namespace NEO diff --git a/shared/source/os_interface/linux/sys_calls_linux.cpp b/shared/source/os_interface/linux/sys_calls_linux.cpp index 28125903e9cf2..2d37911afe7f9 100644 --- a/shared/source/os_interface/linux/sys_calls_linux.cpp +++ b/shared/source/os_interface/linux/sys_calls_linux.cpp @@ -8,14 +8,17 @@ #include "shared/source/os_interface/linux/sys_calls.h" #include +#include #include #include #include #include #include +#include #include #include #include +#include #include namespace NEO { @@ -43,6 +46,9 @@ int close(int fileDescriptor) { int open(const char *file, int flags) { return ::open(file, flags); } +int openWithMode(const char *file, int flags, int mode) { + return ::open(file, flags, mode); +} int ioctl(int fileDescriptor, unsigned long int request, void *arg) { return ::ioctl(fileDescriptor, request, arg); } @@ -114,9 +120,37 @@ char *realpath(const char *path, char *buf) { return ::realpath(path, buf); } +int stat(const std::string &filePath, struct stat *statbuf) { + return ::stat(filePath.c_str(), statbuf); +} + int pipe(int pipeFd[2]) { return ::pipe(pipeFd); } +int mkstemp(char *filePath) { + return ::mkstemp(filePath); +} + +int flock(int fd, int flag) { + return ::flock(fd, flag); +} + +int rename(const char *currName, const char *dstName) { + return ::rename(currName, dstName); +} + +int scandir(const char *dirp, + struct dirent ***namelist, + int (*filter)(const struct dirent *), + int (*compar)(const struct dirent **, + const struct dirent **)) { + return ::scandir(dirp, namelist, filter, compar); +} + +int unlink(const std::string &pathname) { + return ::unlink(pathname.c_str()); +} + } // namespace SysCalls } // namespace NEO diff --git a/shared/test/common/mocks/mock_compiler_cache.h b/shared/test/common/mocks/mock_compiler_cache.h index 91f2436f09275..8a0087f41b280 100644 --- a/shared/test/common/mocks/mock_compiler_cache.h +++ b/shared/test/common/mocks/mock_compiler_cache.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 Intel Corporation + * Copyright (C) 2022-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -15,12 +15,12 @@ class CompilerCacheMock : public CompilerCache { CompilerCacheMock() : CompilerCache(CompilerCacheConfig{}) { } - bool cacheBinary(const std::string kernelFileHash, const char *pBinary, uint32_t binarySize) override { + bool cacheBinary(const std::string &kernelFileHash, const char *pBinary, size_t binarySize) override { cacheInvoked++; return cacheResult; } - std::unique_ptr loadCachedBinary(const std::string kernelFileHash, size_t &cachedBinarySize) override { + std::unique_ptr loadCachedBinary(const std::string &kernelFileHash, size_t &cachedBinarySize) override { if (loadResult || numberOfLoadResult > 0) { numberOfLoadResult--; cachedBinarySize = sizeof(char); diff --git a/shared/test/common/os_interface/linux/sys_calls_linux_ult.cpp b/shared/test/common/os_interface/linux/sys_calls_linux_ult.cpp index 87b65cf4b5875..192f787a06c6d 100644 --- a/shared/test/common/os_interface/linux/sys_calls_linux_ult.cpp +++ b/shared/test/common/os_interface/linux/sys_calls_linux_ult.cpp @@ -17,11 +17,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -40,6 +42,7 @@ bool allowFakeDevicePath = false; constexpr unsigned long int invalidIoctl = static_cast(-1); int setErrno = 0; int fstatFuncRetVal = 0; +int flockRetVal = 0; uint32_t preadFuncCalled = 0u; uint32_t pwriteFuncCalled = 0u; bool isInvalidAILTest = false; @@ -47,6 +50,12 @@ const char *drmVersion = "i915"; int passedFileDescriptorFlagsToSet = 0; int getFileDescriptorFlagsCalled = 0; int setFileDescriptorFlagsCalled = 0; +int unlinkCalled = 0; +int scandirCalled = 0; +int mkstempCalled = 0; +int renameCalled = 0; +int pathFileExistsCalled = 0; +int flockCalled = 0; std::vector mmapVector(64); std::vector mmapCapturedExtendedPointers(64); @@ -57,6 +66,8 @@ uint32_t mmapFuncCalled = 0u; uint32_t munmapFuncCalled = 0u; int (*sysCallsOpen)(const char *pathname, int flags) = nullptr; +int (*sysCallsClose)(int fileDescriptor) = nullptr; +int (*sysCallsOpenWithMode)(const char *pathname, int flags, int mode) = nullptr; ssize_t (*sysCallsPread)(int fd, void *buf, size_t count, off_t offset) = nullptr; ssize_t (*sysCallsPwrite)(int fd, const void *buf, size_t count, off_t offset) = nullptr; int (*sysCallsReadlink)(const char *path, char *buf, size_t bufsize) = nullptr; @@ -67,6 +78,15 @@ ssize_t (*sysCallsWrite)(int fd, void *buf, size_t count) = nullptr; int (*sysCallsPipe)(int pipeFd[2]) = nullptr; int (*sysCallsFstat)(int fd, struct stat *buf) = nullptr; char *(*sysCallsRealpath)(const char *path, char *buf) = nullptr; +int (*sysCallsRename)(const char *currName, const char *dstName); +int (*sysCallsScandir)(const char *dirp, + struct dirent ***namelist, + int (*filter)(const struct dirent *), + int (*compar)(const struct dirent **, + const struct dirent **)) = nullptr; +int (*sysCallsUnlink)(const std::string &pathname) = nullptr; +int (*sysCallsStat)(const std::string &filePath, struct stat *statbuf) = nullptr; +int (*sysCallsMkstemp)(char *fileName) = nullptr; void exit(int code) { exitCalled = true; @@ -91,6 +111,22 @@ int open(const char *file, int flags) { if (strcmp(file, NEO_SHARED_TEST_FILES_DIR "/linux/by-path/pci-0000:00:02.0-render") == 0) { return fakeFileDescriptor; } + std::string_view configFile = file; + if (configFile.find("config.file") != configFile.npos) { + return fakeFileDescriptor; + } + + return 0; +} + +int openWithMode(const char *file, int flags, int mode) { + if (sysCallsOpenWithMode != nullptr) { + return sysCallsOpenWithMode(file, flags, mode); + } + std::string_view configFile = file; + if (configFile.find("config.file") != configFile.npos) { + return fakeFileDescriptor; + } return 0; } @@ -288,5 +324,66 @@ char *realpath(const char *path, char *buf) { return nullptr; } +int mkstemp(char *fileName) { + mkstempCalled++; + + if (sysCallsMkstemp != nullptr) { + return sysCallsMkstemp(fileName); + } + + return 0; +} + +int flock(int fd, int flag) { + flockCalled++; + + if (fd > 0 && flockRetVal == 0) { + return 0; + } + + return -1; +} +int rename(const char *currName, const char *dstName) { + renameCalled++; + + if (sysCallsRename != nullptr) { + return sysCallsRename(currName, dstName); + } + + return 0; +} + +int scandir(const char *dirp, + struct dirent ***namelist, + int (*filter)(const struct dirent *), + int (*compar)(const struct dirent **, + const struct dirent **)) { + scandirCalled++; + + if (sysCallsScandir != nullptr) { + return sysCallsScandir(dirp, namelist, filter, compar); + } + + return 0; +} + +int unlink(const std::string &pathname) { + unlinkCalled++; + + if (sysCallsUnlink != nullptr) { + return sysCallsUnlink(pathname); + } + + return 0; +} + +int stat(const std::string &filePath, struct stat *statbuf) { + if (sysCallsStat != nullptr) { + return sysCallsStat(filePath, statbuf); + } + + return 0; +} + } // namespace SysCalls } // namespace NEO diff --git a/shared/test/common/os_interface/linux/sys_calls_linux_ult.h b/shared/test/common/os_interface/linux/sys_calls_linux_ult.h index 94a1be2bf9894..efbf1467c5047 100644 --- a/shared/test/common/os_interface/linux/sys_calls_linux_ult.h +++ b/shared/test/common/os_interface/linux/sys_calls_linux_ult.h @@ -7,6 +7,7 @@ #pragma once +#include #include #include #include @@ -16,6 +17,7 @@ namespace NEO { namespace SysCalls { extern int (*sysCallsOpen)(const char *pathname, int flags); +extern int (*sysCallsOpenWithMode)(const char *pathname, int flags, int mode); extern ssize_t (*sysCallsPread)(int fd, void *buf, size_t count, off_t offset); extern ssize_t (*sysCallsPwrite)(int fd, const void *buf, size_t count, off_t offset); extern int (*sysCallsReadlink)(const char *path, char *buf, size_t bufsize); @@ -26,7 +28,20 @@ extern ssize_t (*sysCallsWrite)(int fd, void *buf, size_t count); extern int (*sysCallsPipe)(int pipeFd[2]); extern int (*sysCallsFstat)(int fd, struct stat *buf); extern char *(*sysCallsRealpath)(const char *path, char *buf); +extern ssize_t (*sysCallsPwrite)(int fd, const void *buf, size_t count, off_t offset); +extern int (*sysCallsRename)(const char *currName, const char *dstName); +extern int (*sysCallsScandir)(const char *dirp, + struct dirent ***namelist, + int (*filter)(const struct dirent *), + int (*compar)(const struct dirent **, + const struct dirent **)); +extern int (*sysCallsUnlink)(const std::string &pathname); +extern int (*sysCallsStat)(const std::string &filePath, struct stat *statbuf); +extern int (*sysCallsMkstemp)(char *fileName); +extern int flockRetVal; +extern int closeFuncRetVal; +extern int closeFuncArgPassed; extern const char *drmVersion; constexpr int fakeFileDescriptor = 123; extern int passedFileDescriptorFlagsToSet; @@ -35,6 +50,12 @@ extern int setFileDescriptorFlagsCalled; extern uint32_t closeFuncCalled; extern bool exitCalled; extern int latestExitCode; +extern int unlinkCalled; +extern int scandirCalled; +extern int mkstempCalled; +extern int renameCalled; +extern int pathFileExistsCalled; +extern int flockCalled; extern std::vector mmapVector; extern std::vector mmapCapturedExtendedPointers; diff --git a/shared/test/unit_test/compiler_interface/compiler_cache_tests.cpp b/shared/test/unit_test/compiler_interface/compiler_cache_tests.cpp index 086fa5b74498c..d6fa280e21ac4 100644 --- a/shared/test/unit_test/compiler_interface/compiler_cache_tests.cpp +++ b/shared/test/unit_test/compiler_interface/compiler_cache_tests.cpp @@ -418,3 +418,573 @@ TEST(CompilerInterfaceCachedTests, givenKernelWithIncludesAndBinaryInCacheWhenCo gEnvironment->fclPopDebugVars(); } + +#ifndef _WIN32 + +#include "shared/test/common/os_interface/linux/sys_calls_linux_ult.h" + +class CompilerCacheMockLinux : public CompilerCache { + public: + CompilerCacheMockLinux(const CompilerCacheConfig &config) : CompilerCache(config) {} + using CompilerCache::createUniqueTempFileAndWriteData; + using CompilerCache::evictCache; + using CompilerCache::lockConfigFileAndReadSize; + using CompilerCache::renameTempFileBinaryToProperName; +}; + +namespace EvictCachePass { +decltype(NEO::SysCalls::sysCallsScandir) mockScandir = [](const char *dirp, + struct dirent ***namelist, + int (*filter)(const struct dirent *), + int (*compar)(const struct dirent **, + const struct dirent **)) -> int { + struct dirent **v = (struct dirent **)malloc(6 * (sizeof(struct dirent *))); + + v[0] = (struct dirent *)malloc(sizeof(struct dirent)); + + memcpy_s(v[0]->d_name, sizeof(v[0]->d_name), "file1.cl_cache", sizeof("file1.cl_cache")); + + v[1] = (struct dirent *)malloc(sizeof(struct dirent)); + + memcpy_s(v[1]->d_name, sizeof(v[1]->d_name), "file2.cl_cache", sizeof("file2.cl_cache")); + + v[2] = (struct dirent *)malloc(sizeof(struct dirent)); + + memcpy_s(v[2]->d_name, sizeof(v[2]->d_name), "file3.cl_cache", sizeof("file3.cl_cache")); + + v[3] = (struct dirent *)malloc(sizeof(struct dirent)); + + memcpy_s(v[3]->d_name, sizeof(v[3]->d_name), "file4.cl_cache", sizeof("file4.cl_cache")); + + v[4] = (struct dirent *)malloc(sizeof(struct dirent)); + + memcpy_s(v[4]->d_name, sizeof(v[4]->d_name), "file5.cl_cache", sizeof("file5.cl_cache")); + + v[5] = (struct dirent *)malloc(sizeof(struct dirent)); + + memcpy_s(v[5]->d_name, sizeof(v[5]->d_name), "file6.cl_cache", sizeof("file6.cl_cache")); + + *namelist = v; + return 6; +}; + +decltype(NEO::SysCalls::sysCallsStat) mockStat = [](const std::string &filePath, struct stat *statbuf) -> int { + if (filePath.find("file1") != filePath.npos) { + statbuf->st_atime = 3; + } + if (filePath.find("file2") != filePath.npos) { + statbuf->st_atime = 6; + } + if (filePath.find("file3") != filePath.npos) { + statbuf->st_atime = 1; + } + if (filePath.find("file4") != filePath.npos) { + statbuf->st_atime = 2; + } + if (filePath.find("file5") != filePath.npos) { + statbuf->st_atime = 4; + } + if (filePath.find("file6") != filePath.npos) { + statbuf->st_atime = 5; + } + statbuf->st_size = (MemoryConstants::megaByte / 6) + 10; + + return 0; +}; + +std::vector *unlinkFiles; + +decltype(NEO::SysCalls::sysCallsUnlink) mockUnlink = [](const std::string &pathname) -> int { + unlinkFiles->push_back(pathname); + + return 0; +}; +} // namespace EvictCachePass + +TEST(CompilerCacheTests, GivenCompilerCacheWithOneMegabyteWhenEvictCacheIsCalledThenUnlinkTwoOldestFiles) { + std::vector unlinkLocalFiles; + EvictCachePass::unlinkFiles = &unlinkLocalFiles; + + VariableBackup scandirBackup(&NEO::SysCalls::sysCallsScandir, EvictCachePass::mockScandir); + VariableBackup statBackup(&NEO::SysCalls::sysCallsStat, EvictCachePass::mockStat); + VariableBackup unlinkBackup(&NEO::SysCalls::sysCallsUnlink, EvictCachePass::mockUnlink); + + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte - 2u}); + + EXPECT_TRUE(cache.evictCache()); + + EXPECT_EQ(unlinkLocalFiles.size(), 2u); + + EXPECT_NE(unlinkLocalFiles[0].find("file3"), unlinkLocalFiles[0].npos); + EXPECT_NE(unlinkLocalFiles[1].find("file4"), unlinkLocalFiles[1].npos); +} + +TEST(CompilerCacheTests, GivenCompilerCacheWithWhenScandirFailThenEvictCacheFail) { + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + VariableBackup scandirBackup(&NEO::SysCalls::sysCallsScandir, [](const char *dirp, struct dirent ***namelist, int (*filter)(const struct dirent *), int (*compar)(const struct dirent **, const struct dirent **)) -> int { return -1; }); + + EXPECT_FALSE(cache.evictCache()); +} + +namespace CreateUniqueTempFilePass { +decltype(NEO::SysCalls::sysCallsMkstemp) mockMkstemp = [](char *fileName) -> int { + memcpy_s(&fileName[22], 20, "123456", sizeof("123456")); + return 1; +}; + +decltype(NEO::SysCalls::sysCallsPwrite) mockPwrite = [](int fd, const void *buf, size_t count, off_t offset) -> ssize_t { + return count; +}; +} // namespace CreateUniqueTempFilePass + +TEST(CompilerCacheTests, GivenCompilerCacheWhenCreateUniqueTempFileAndWriteDataThenFileIsCreatedAndDataAreWritten) { + VariableBackup mkstempBackup(&NEO::SysCalls::sysCallsMkstemp, CreateUniqueTempFilePass::mockMkstemp); + VariableBackup pwriteBackup(&NEO::SysCalls::sysCallsPwrite, CreateUniqueTempFilePass::mockPwrite); + + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte - 2u}); + + char x[256] = "/home/cl_cache/mytemp.XXXXXX"; + char compare[256] = "/home/cl_cache/mytemp.123456"; + EXPECT_TRUE(cache.createUniqueTempFileAndWriteData(x, "1", 1)); + + EXPECT_EQ(0, memcmp(x, compare, 28)); +} + +namespace CreateUniqueTempFileFail1 { +decltype(NEO::SysCalls::sysCallsMkstemp) mockMkstemp = [](char *fileName) -> int { + return -1; +}; +} // namespace CreateUniqueTempFileFail1 + +TEST(CompilerCacheTests, GivenCompilerCacheWithCreateUniqueTempFileAndWriteDataWhenMkstempFailThenFalseIsReturned) { + VariableBackup mkstempBackup(&NEO::SysCalls::sysCallsMkstemp, CreateUniqueTempFileFail1::mockMkstemp); + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte - 2u}); + + char x[256] = "/home/cl_cache/mytemp.XXXXXX"; + EXPECT_FALSE(cache.createUniqueTempFileAndWriteData(x, "1", 1)); +} + +namespace CreateUniqueTempFileFail2 { +decltype(NEO::SysCalls::sysCallsMkstemp) mockMkstemp = [](char *fileName) -> int { + memcpy_s(&fileName[22], 20, "123456", sizeof("123456")); + return 1; +}; + +decltype(NEO::SysCalls::sysCallsPwrite) mockPwrite = [](int fd, const void *buf, size_t count, off_t offset) -> ssize_t { + return -1; +}; +} // namespace CreateUniqueTempFileFail2 + +TEST(CompilerCacheTests, GivenCompilerCacheWithCreateUniqueTempFileAndWriteDataWhenPwriteFailThenFalseIsReturnedAndCleanupIsMade) { + std::vector unlinkLocalFiles; + EvictCachePass::unlinkFiles = &unlinkLocalFiles; + + VariableBackup unlinkBackup(&NEO::SysCalls::sysCallsUnlink, EvictCachePass::mockUnlink); + VariableBackup mkstempBackup(&NEO::SysCalls::sysCallsMkstemp, CreateUniqueTempFileFail2::mockMkstemp); + VariableBackup pwriteBackup(&NEO::SysCalls::sysCallsPwrite, CreateUniqueTempFileFail2::mockPwrite); + + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte - 2u}); + + char x[256] = "/home/cl_cache/mytemp.XXXXXX"; + EXPECT_FALSE(cache.createUniqueTempFileAndWriteData(x, "1", 1)); + EXPECT_EQ(NEO::SysCalls::closeFuncArgPassed, 1); + EXPECT_EQ(0, memcmp(x, unlinkLocalFiles[0].c_str(), 28)); +} + +TEST(CompilerCacheTests, GivenCompilerCacheWithCreateUniqueTempFileAndWriteDataWhenCloseFailThenFalseIsReturned) { + int closeRetVal = -1; + VariableBackup mkstempBackup(&NEO::SysCalls::sysCallsMkstemp, CreateUniqueTempFilePass::mockMkstemp); + VariableBackup pwriteBackup(&NEO::SysCalls::sysCallsPwrite, CreateUniqueTempFilePass::mockPwrite); + VariableBackup closeBackup(&NEO::SysCalls::closeFuncRetVal, closeRetVal); + + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte - 2u}); + + char x[256] = "/home/cl_cache/mytemp.XXXXXX"; + EXPECT_FALSE(cache.createUniqueTempFileAndWriteData(x, "1", 1)); +} + +TEST(CompilerCacheTests, GivenCompilerCacheWhenRenameFailThenFalseIsReturned) { + int unlinkTemp = 0; + VariableBackup unlinkBackup(&NEO::SysCalls::unlinkCalled, unlinkTemp); + VariableBackup renameBackup(&NEO::SysCalls::sysCallsRename, [](const char *currName, const char *dstName) -> int { return -1; }); + + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + + EXPECT_FALSE(cache.renameTempFileBinaryToProperName("src", "dst")); + EXPECT_EQ(NEO::SysCalls::unlinkCalled, 1); +} + +TEST(CompilerCacheTests, GivenCompilerCacheWhenRenameThenTrueIsReturned) { + VariableBackup renameBackup(&NEO::SysCalls::sysCallsRename, [](const char *currName, const char *dstName) -> int { return 0; }); + + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + + EXPECT_TRUE(cache.renameTempFileBinaryToProperName("src", "dst")); +} + +TEST(CompilerCacheTests, GivenCompilerCacheWhenConfigFileIsInacessibleThenFdIsSetToNegativeNumber) { + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + + VariableBackup openBackup(&NEO::SysCalls::sysCallsOpen, [](const char *pathname, int flags) -> int { errno = EACCES; return -1; }); + + int configFileDescriptor = 0; + size_t directory = 0; + + cache.lockConfigFileAndReadSize("config.file", configFileDescriptor, directory); + + EXPECT_EQ(configFileDescriptor, -1); +} + +TEST(CompilerCacheTests, GivenCompilerCacheWhenCannotLockConfigFileThenFdIsSetToNegativeNumber) { + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + int flockRetVal = -1; + + VariableBackup flockBackup(&NEO::SysCalls::flockRetVal, flockRetVal); + + int configFileDescriptor = 0; + size_t directory = 0; + + cache.lockConfigFileAndReadSize("config.file", configFileDescriptor, directory); + + EXPECT_EQ(configFileDescriptor, -1); +} + +#include + +namespace LockConfigFileAndReadSize { +int openWithMode(const char *file, int flags, int mode) { + + if (flags == (O_CREAT | O_EXCL | O_RDWR)) { + return 1; + } + + errno = 2; + return -1; +} + +int open(const char *file, int flags) { + errno = 2; + return -1; +} +} // namespace LockConfigFileAndReadSize + +TEST(CompilerCacheTests, GivenCompilerCacheWhenScandirFailInLockConfigFileThenFdIsSetToNegativeNumber) { + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + + VariableBackup scandirBackup(&NEO::SysCalls::sysCallsScandir, [](const char *dirp, struct dirent ***namelist, int (*filter)(const struct dirent *), int (*compar)(const struct dirent **, const struct dirent **)) -> int { return -1; }); + VariableBackup openWithModeBackup(&NEO::SysCalls::sysCallsOpenWithMode, LockConfigFileAndReadSize::openWithMode); + VariableBackup openBackup(&NEO::SysCalls::sysCallsOpen, LockConfigFileAndReadSize::open); + + int configFileDescriptor = 0; + size_t directory = 0; + + cache.lockConfigFileAndReadSize("config.file", configFileDescriptor, directory); + + EXPECT_EQ(configFileDescriptor, -1); +} + +namespace lockConfigFileAndReadSizeMocks { +decltype(NEO::SysCalls::sysCallsScandir) mockScandir = [](const char *dirp, + struct dirent ***namelist, + int (*filter)(const struct dirent *), + int (*compar)(const struct dirent **, + const struct dirent **)) -> int { + struct dirent **v = (struct dirent **)malloc(4 * (sizeof(struct dirent *))); + + v[0] = (struct dirent *)malloc(sizeof(struct dirent)); + + memcpy_s(v[0]->d_name, sizeof(v[0]->d_name), "file1.cl_cache", sizeof("file1.cl_cache")); + + v[1] = (struct dirent *)malloc(sizeof(struct dirent)); + + memcpy_s(v[1]->d_name, sizeof(v[1]->d_name), "file2.cl_cache", sizeof("file2.cl_cache")); + + v[2] = (struct dirent *)malloc(sizeof(struct dirent)); + + memcpy_s(v[2]->d_name, sizeof(v[2]->d_name), "file3.cl_cache", sizeof("file3.cl_cache")); + + v[3] = (struct dirent *)malloc(sizeof(struct dirent)); + + memcpy_s(v[3]->d_name, sizeof(v[3]->d_name), "file4.cl_cache", sizeof("file4.cl_cache")); + + *namelist = v; + return 4; +}; + +decltype(NEO::SysCalls::sysCallsStat) mockStat = [](const std::string &filePath, struct stat *statbuf) -> int { + if (filePath.find("file1") != filePath.npos) { + statbuf->st_atime = 3; + } + if (filePath.find("file2") != filePath.npos) { + statbuf->st_atime = 6; + } + if (filePath.find("file3") != filePath.npos) { + statbuf->st_atime = 1; + } + if (filePath.find("file4") != filePath.npos) { + statbuf->st_atime = 2; + } + + statbuf->st_size = (MemoryConstants::megaByte / 4); + + return 0; +}; +} // namespace lockConfigFileAndReadSizeMocks + +TEST(CompilerCacheTests, GivenCompilerCacheWhenLockConfigFileWithFileCreationThenFdIsSetProperSizeIsSetAndScandirIsCalled) { + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + int scandirCalledTemp = 0; + + VariableBackup scandirCalledBackup(&NEO::SysCalls::scandirCalled, scandirCalledTemp); + VariableBackup scandirBackup(&NEO::SysCalls::sysCallsScandir, lockConfigFileAndReadSizeMocks::mockScandir); + VariableBackup statBackup(&NEO::SysCalls::sysCallsStat, lockConfigFileAndReadSizeMocks::mockStat); + VariableBackup openWithModeBackup(&NEO::SysCalls::sysCallsOpenWithMode, LockConfigFileAndReadSize::openWithMode); + VariableBackup openBackup(&NEO::SysCalls::sysCallsOpen, LockConfigFileAndReadSize::open); + + int configFileDescriptor = 0; + size_t directory = 0; + + cache.lockConfigFileAndReadSize("config.file", configFileDescriptor, directory); + + EXPECT_EQ(configFileDescriptor, 1); + EXPECT_EQ(NEO::SysCalls::scandirCalled, 1); + EXPECT_EQ(directory, MemoryConstants::megaByte); +} + +namespace LockConfigFileAndConfigFileIsCreatedInMeantime { +int openCalledTimes = 0; +int openWithMode(const char *file, int flags, int mode) { + + if (flags == (O_CREAT | O_EXCL | O_RDWR)) { + return -1; + } + + return 0; +} + +int open(const char *file, int flags) { + if (openCalledTimes == 0) { + openCalledTimes++; + errno = 2; + return -1; + } + + return 1; +} + +size_t configSize = MemoryConstants::megaByte; +ssize_t pread(int fd, void *buf, size_t count, off_t offset) { + memcpy(buf, &configSize, sizeof(configSize)); + return 0; +} +} // namespace LockConfigFileAndConfigFileIsCreatedInMeantime + +TEST(CompilerCacheTests, GivenCompilerCacheWhenLockConfigFileAndOtherProcessCreateInMeanTimeThenFdIsSetProperSizeIsSetAndScandirIsNotCalled) { + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + int openCalledTimes = 0; + int scandirCalledTimes = 0; + + VariableBackup scandirCalledBackup(&NEO::SysCalls::scandirCalled, scandirCalledTimes); + VariableBackup openCalledBackup(&LockConfigFileAndConfigFileIsCreatedInMeantime::openCalledTimes, openCalledTimes); + VariableBackup preadBackup(&NEO::SysCalls::sysCallsPread, LockConfigFileAndConfigFileIsCreatedInMeantime::pread); + VariableBackup openWithModeBackup(&NEO::SysCalls::sysCallsOpenWithMode, LockConfigFileAndConfigFileIsCreatedInMeantime::openWithMode); + VariableBackup openBackup(&NEO::SysCalls::sysCallsOpen, LockConfigFileAndConfigFileIsCreatedInMeantime::open); + + int configFileDescriptor = 0; + size_t directory = 0; + + cache.lockConfigFileAndReadSize("config.file", configFileDescriptor, directory); + + EXPECT_EQ(configFileDescriptor, 1); + EXPECT_EQ(NEO::SysCalls::scandirCalled, 0); + EXPECT_EQ(directory, LockConfigFileAndConfigFileIsCreatedInMeantime::configSize); +} + +TEST(CompilerCacheTests, GivenCompilerCacheWhenLockConfigFileThenFdIsSetProperSizeIsSetAndScandirIsNotCalled) { + CompilerCacheMockLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + int scandirCalledTimes = 0; + + VariableBackup scandirCalledBackup(&NEO::SysCalls::scandirCalled, scandirCalledTimes); + VariableBackup preadBackup(&NEO::SysCalls::sysCallsPread, LockConfigFileAndConfigFileIsCreatedInMeantime::pread); + + int configFileDescriptor = 0; + size_t directory = 0; + + cache.lockConfigFileAndReadSize("config.file", configFileDescriptor, directory); + + EXPECT_EQ(configFileDescriptor, NEO::SysCalls::fakeFileDescriptor); + EXPECT_EQ(NEO::SysCalls::scandirCalled, 0); + EXPECT_EQ(directory, LockConfigFileAndConfigFileIsCreatedInMeantime::configSize); +} + +class CompilerCacheFailingLokcConfigFileAndReadSizeLinux : public CompilerCache { + public: + CompilerCacheFailingLokcConfigFileAndReadSizeLinux(const CompilerCacheConfig &config) : CompilerCache(config) {} + using CompilerCache::createUniqueTempFileAndWriteData; + using CompilerCache::evictCache; + using CompilerCache::lockConfigFileAndReadSize; + using CompilerCache::renameTempFileBinaryToProperName; + + void lockConfigFileAndReadSize(const std::string &configFilePath, int &fd, size_t &directorySize) override { + fd = -1; + return; + } +}; + +TEST(CompilerCacheTests, GivenCompilerCacheWhenLockConfigFileFailThenCacheBinaryReturnsFalse) { + CompilerCacheFailingLokcConfigFileAndReadSizeLinux cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + + EXPECT_FALSE(cache.cacheBinary("config.file", "1", 1)); +} + +class CompilerCacheLinuxReturnTrueIfAnotherProcessCreateCacheFile : public CompilerCache { + public: + CompilerCacheLinuxReturnTrueIfAnotherProcessCreateCacheFile(const CompilerCacheConfig &config) : CompilerCache(config) {} + using CompilerCache::createUniqueTempFileAndWriteData; + using CompilerCache::evictCache; + using CompilerCache::lockConfigFileAndReadSize; + using CompilerCache::renameTempFileBinaryToProperName; + + void lockConfigFileAndReadSize(const std::string &configFilePath, int &fd, size_t &directorySize) override { + fd = 1; + return; + } +}; + +TEST(CompilerCacheTests, GivenCompilerCacheWhenFileAlreadyExistsThenCacheBinaryReturnsTrue) { + VariableBackup statBackup(&NEO::SysCalls::sysCallsStat, [](const std::string &filePath, struct stat *statbuf) -> int { return 0; }); + + CompilerCacheLinuxReturnTrueIfAnotherProcessCreateCacheFile cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + + EXPECT_TRUE(cache.cacheBinary("config.file", "1", 1)); +} + +class CompilerCacheLinuxReturnFalseOnCacheBinaryIfEvictFailed : public CompilerCache { + public: + CompilerCacheLinuxReturnFalseOnCacheBinaryIfEvictFailed(const CompilerCacheConfig &config) : CompilerCache(config) {} + using CompilerCache::createUniqueTempFileAndWriteData; + using CompilerCache::evictCache; + using CompilerCache::lockConfigFileAndReadSize; + using CompilerCache::renameTempFileBinaryToProperName; + + void lockConfigFileAndReadSize(const std::string &configFilePath, int &fd, size_t &directorySize) override { + directorySize = MemoryConstants::megaByte; + fd = 1; + return; + } + + bool evictCache() override { + return false; + } +}; + +TEST(CompilerCacheTests, GivenCompilerCacheWhenEvictCacheFailThenCacheBinaryReturnsFalse) { + VariableBackup statBackup(&NEO::SysCalls::sysCallsStat, [](const std::string &filePath, struct stat *statbuf) -> int { return -1; }); + + CompilerCacheLinuxReturnFalseOnCacheBinaryIfEvictFailed cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + + EXPECT_FALSE(cache.cacheBinary("config.file", "1", 1)); +} + +class CompilerCacheLinuxReturnFalseOnCacheBinaryIfCreateUniqueFileFailed : public CompilerCache { + public: + CompilerCacheLinuxReturnFalseOnCacheBinaryIfCreateUniqueFileFailed(const CompilerCacheConfig &config) : CompilerCache(config) {} + using CompilerCache::createUniqueTempFileAndWriteData; + using CompilerCache::evictCache; + using CompilerCache::lockConfigFileAndReadSize; + using CompilerCache::renameTempFileBinaryToProperName; + + void lockConfigFileAndReadSize(const std::string &configFilePath, int &fd, size_t &directorySize) override { + directorySize = MemoryConstants::megaByte; + fd = 1; + return; + } + + bool evictCache() override { + return true; + } + + bool createUniqueTempFileAndWriteData(char *tmpFilePathTemplate, const char *pBinary, size_t binarySize) override { + return false; + } +}; + +TEST(CompilerCacheTests, GivenCompilerCacheWhenCreateUniqueFileFailThenCacheBinaryReturnsFalse) { + VariableBackup statBackup(&NEO::SysCalls::sysCallsStat, [](const std::string &filePath, struct stat *statbuf) -> int { return -1; }); + + CompilerCacheLinuxReturnFalseOnCacheBinaryIfCreateUniqueFileFailed cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + + EXPECT_FALSE(cache.cacheBinary("config.file", "1", 1)); +} + +class CompilerCacheLinuxReturnFalseOnCacheBinaryIfRenameFileFailed : public CompilerCache { + public: + CompilerCacheLinuxReturnFalseOnCacheBinaryIfRenameFileFailed(const CompilerCacheConfig &config) : CompilerCache(config) {} + using CompilerCache::createUniqueTempFileAndWriteData; + using CompilerCache::evictCache; + using CompilerCache::lockConfigFileAndReadSize; + using CompilerCache::renameTempFileBinaryToProperName; + + void lockConfigFileAndReadSize(const std::string &configFilePath, int &fd, size_t &directorySize) override { + directorySize = MemoryConstants::megaByte; + fd = 1; + return; + } + + bool evictCache() override { + return true; + } + + bool createUniqueTempFileAndWriteData(char *tmpFilePathTemplate, const char *pBinary, size_t binarySize) override { + return true; + } + + bool renameTempFileBinaryToProperName(const std::string &oldName, const std::string &kernelFileHash) override { + return false; + } +}; + +TEST(CompilerCacheTests, GivenCompilerCacheWhenRenameFileFailThenCacheBinaryReturnsFalse) { + VariableBackup statBackup(&NEO::SysCalls::sysCallsStat, [](const std::string &filePath, struct stat *statbuf) -> int { return -1; }); + + CompilerCacheLinuxReturnFalseOnCacheBinaryIfRenameFileFailed cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + + EXPECT_FALSE(cache.cacheBinary("config.file", "1", 1)); +} + +class CompilerCacheLinuxReturnTrueOnCacheBinary : public CompilerCache { + public: + CompilerCacheLinuxReturnTrueOnCacheBinary(const CompilerCacheConfig &config) : CompilerCache(config) {} + using CompilerCache::createUniqueTempFileAndWriteData; + using CompilerCache::evictCache; + using CompilerCache::lockConfigFileAndReadSize; + using CompilerCache::renameTempFileBinaryToProperName; + + void lockConfigFileAndReadSize(const std::string &configFilePath, int &fd, size_t &directorySize) override { + directorySize = MemoryConstants::megaByte; + fd = 1; + return; + } + + bool evictCache() override { + return true; + } + + bool createUniqueTempFileAndWriteData(char *tmpFilePathTemplate, const char *pBinary, size_t binarySize) override { + return true; + } + + bool renameTempFileBinaryToProperName(const std::string &oldName, const std::string &kernelFileHash) override { + return true; + } +}; + +TEST(CompilerCacheTests, GivenCompilerCacheWhenAllFunctionsSuccedThenCacheBinaryReturnsTrue) { + VariableBackup statBackup(&NEO::SysCalls::sysCallsStat, [](const std::string &filePath, struct stat *statbuf) -> int { return 0; }); + + CompilerCacheLinuxReturnTrueOnCacheBinary cache({true, ".cl_cache", "/home/cl_cache/", MemoryConstants::megaByte}); + + EXPECT_TRUE(cache.cacheBinary("config.file", "1", 1)); +} + +#endif // !_WIN32