From 22a348dc19e37387ebd1fce7bbc775de0afd5e5b Mon Sep 17 00:00:00 2001 From: Roman Koshelev Date: Wed, 2 Dec 2020 21:24:04 +0300 Subject: [PATCH] Rewritten file I / O for portable C API Besides portability, it also provides more guarantees (for example, the write system call does not guarantee that the entire buffer will be written, unlike fwrite, which tries to write the remainder). --- runtime/Config.h | 2 +- runtime/DoubleBuffer.h | 23 +++---------- runtime/FSyncPortable.h | 27 +++++++++++++++ runtime/RuntimeLogger.cc | 72 ++++++++++++++++------------------------ runtime/RuntimeLogger.h | 7 ++-- 5 files changed, 63 insertions(+), 68 deletions(-) create mode 100644 runtime/FSyncPortable.h diff --git a/runtime/Config.h b/runtime/Config.h index 3ee6d87..28ee813 100644 --- a/runtime/Config.h +++ b/runtime/Config.h @@ -25,7 +25,7 @@ namespace NanoLogConfig { // Controls in what mode the compressed log file will be opened - static const int FILE_PARAMS = O_APPEND|O_RDWR|O_CREAT|O_NOATIME|O_DSYNC; + static const char FILE_PARAMS[] = "a+"; // Location of the initial log file static const char DEFAULT_LOG_FILE[] = "./compressedLog"; diff --git a/runtime/DoubleBuffer.h b/runtime/DoubleBuffer.h index 0eddbbc..cc8b0fa 100644 --- a/runtime/DoubleBuffer.h +++ b/runtime/DoubleBuffer.h @@ -13,24 +13,11 @@ class DoubleBuffer { - struct Deleter { - void operator()(char *__ptr) const { free(__ptr); } - }; - - using MemPtrT = std::unique_ptr; + using MemPtrT = std::unique_ptr; static MemPtrT allocateMemory() { - char *tmp; - int err = posix_memalign(reinterpret_cast(&tmp), 512, - NanoLogConfig::OUTPUT_BUFFER_SIZE); - if (err) { - perror( - "The NanoLog system was not able to allocate enough memory " - "to support its operations. Quitting...\r\n"); - std::exit(-1); - } - return MemPtrT{tmp}; + return MemPtrT{new char[NanoLogConfig::OUTPUT_BUFFER_SIZE]}; }; static char *accessOnce(const MemPtrT &ptr) @@ -80,7 +67,7 @@ class DoubleBuffer } } - void writeToFile(int file) noexcept + void writeToFile(FILE* file) noexcept { unsigned tmp_size = 0; MemPtrT tmp_ptr = nullptr; @@ -94,8 +81,8 @@ class DoubleBuffer int res = 0; if (tmp_size != 0) { - res = write(file, tmp_ptr.get(), tmp_size); - res = (res < 0) ? errno : 0; + size_t wsize = fwrite(tmp_ptr.get(), 1u, tmp_size, file); + res = (wsize != tmp_size) ? errno : 0; } while (accessOnce(freeBuffer) != nullptr) {} diff --git a/runtime/FSyncPortable.h b/runtime/FSyncPortable.h new file mode 100644 index 0000000..4f2b464 --- /dev/null +++ b/runtime/FSyncPortable.h @@ -0,0 +1,27 @@ +#pragma once + +#ifndef _WIN32 + +#include + +#else + +#include +#include + +inline int fsync(int fd) { + HANDLE h = (HANDLE)_get_osfhandle(fd); + if (h == INVALID_HANDLE_VALUE) { + return -1; + } + if (!FlushFileBuffers(h)) { + return -1; + } + return 0; +} + +inline int fdatasync(int fd) { + return fsync(fd); +} + +#endif \ No newline at end of file diff --git a/runtime/RuntimeLogger.cc b/runtime/RuntimeLogger.cc index 871ca0f..305532f 100644 --- a/runtime/RuntimeLogger.cc +++ b/runtime/RuntimeLogger.cc @@ -48,7 +48,7 @@ RuntimeLogger::RuntimeLogger() , condMutex() , workAdded() , hintSyncCompleted() - , outputFd(-1) + , outputFile(nullptr) , buffer() , currentLogLevel(NOTICE) , cycleAtThreadStart(0) @@ -60,7 +60,6 @@ RuntimeLogger::RuntimeLogger() , cyclesDiskIO_upperBound(0) , totalBytesRead(0) , totalBytesWritten(0) - , padBytesWritten(0) , logsProcessed(0) , numAioWritesCompleted(0) , coreId(-1) @@ -72,14 +71,15 @@ RuntimeLogger::RuntimeLogger() stagingBufferPeekDist[i] = 0; const char *filename = NanoLogConfig::DEFAULT_LOG_FILE; - outputFd = open(filename, NanoLogConfig::FILE_PARAMS, 0666); - if (outputFd < 0) { + outputFile = fopen(filename, NanoLogConfig::FILE_PARAMS); + if (outputFile == nullptr) { fprintf(stderr, "NanoLog could not open the default file location " "for the log file (\"%s\").\r\n Please check the permissions " "or use NanoLog::setLogFile(const char* filename) to " "specify a different log file.\r\n", filename); std::exit(-1); } + setvbuf(outputFile, nullptr, _IONBF, 0); #ifndef BENCHMARK_DISCARD_ENTRIES_AT_STAGINGBUFFER compressionThread = std::thread(&RuntimeLogger::compressionThreadMain, this); @@ -109,10 +109,10 @@ RuntimeLogger::~RuntimeLogger() { if (nanoLogSingleton.writerThread.joinable()) nanoLogSingleton.writerThread.join(); - if (outputFd > 0) - close(outputFd); + if (outputFile != nullptr) + fclose(outputFile); - outputFd = 0; + outputFile = nullptr; } // Documentation in NanoLog.h @@ -122,7 +122,7 @@ RuntimeLogger::getStats() { char buffer[1024]; // Leaks abstraction, but basically flush so we get all the time uint64_t start = PerfUtils::Cycles::rdtsc(); - fdatasync(nanoLogSingleton.outputFd); + fdatasync(fileno(nanoLogSingleton.outputFile)); uint64_t stop = PerfUtils::Cycles::rdtsc(); nanoLogSingleton.cyclesDiskIO_upperBound += (stop - start); @@ -136,8 +136,6 @@ RuntimeLogger::getStats() { nanoLogSingleton.totalBytesWritten); double totalBytesReadDouble = static_cast( nanoLogSingleton.totalBytesRead); - double padBytesWrittenDouble = static_cast( - nanoLogSingleton.padBytesWritten); double numEventsProcessedDouble = static_cast( nanoLogSingleton.logsProcessed); @@ -197,14 +195,11 @@ RuntimeLogger::getStats() { compressTime * 1.0e9 / numEventsProcessedDouble); out << buffer; - snprintf(buffer, 1024, "The compression ratio was %0.2lf-%0.2lfx " - "(%lu bytes in, %lu bytes out, %lu pad bytes)\n", - 1.0 * totalBytesReadDouble / (totalBytesWrittenDouble - + padBytesWrittenDouble), + snprintf(buffer, 1024, "The compression ratio was %0.2lf " + "(%lu bytes in, %lu bytes out)\n", 1.0 * totalBytesReadDouble / totalBytesWrittenDouble, nanoLogSingleton.totalBytesRead, - nanoLogSingleton.totalBytesWritten, - nanoLogSingleton.padBytesWritten); + nanoLogSingleton.totalBytesWritten); out << buffer; return out.str(); @@ -525,17 +520,6 @@ RuntimeLogger::compressionThreadMain() { if (bytesToWrite == 0) continue; - // Pad the output if necessary - if (NanoLogConfig::FILE_PARAMS & O_DIRECT) { - ssize_t bytesOver = bytesToWrite % 512; - - if (bytesOver != 0) { - memset(buffer.getCompressingBuffer(), 0, 512 - bytesOver); - bytesToWrite = bytesToWrite + 512 - bytesOver; - padBytesWritten += (512 - bytesOver); - } - } - totalBytesWritten += bytesToWrite; uint64_t tmp = PerfUtils::Cycles::rdtsc(); @@ -569,29 +553,29 @@ RuntimeLogger::compressionThreadMain() { void RuntimeLogger::writerThreadMain() { while (!writerThreadShouldExit) { - buffer.writeToFile(outputFd); + buffer.writeToFile(outputFile); } } // Documentation in NanoLog.h void RuntimeLogger::setLogFile_internal(const char *filename) { - // Check if it exists and is readable/writeable - if (access(filename, F_OK) == 0 && access(filename, R_OK | W_OK) != 0) { - std::string err = "Unable to read/write from new log file: "; - err.append(filename); - throw std::ios_base::failure(err); - } - // Try to open the file - int newFd = open(filename, NanoLogConfig::FILE_PARAMS, 0666); - if (newFd < 0) { - std::string err = "Unable to open file new log file: '"; - err.append(filename); - err.append("': "); - err.append(strerror(errno)); + FILE* newFile = fopen(filename, NanoLogConfig::FILE_PARAMS); + if (newFile == nullptr) { + std::string err; + if (errno == EACCES) { + err = "Unable to read/write from new log file: "; + err.append(filename); + } else { + err = "Unable to open file new log file: '"; + err.append(filename); + err.append("': "); + err.append(strerror(errno)); + } throw std::ios_base::failure(err); } + setvbuf(outputFile, nullptr, _IONBF, 0); // Everything seems okay, stop the background thread and change files sync(); @@ -614,9 +598,9 @@ RuntimeLogger::setLogFile_internal(const char *filename) { if (writerThread.joinable()) writerThread.join(); - if (outputFd > 0) - close(outputFd); - outputFd = newFd; + if (outputFile != nullptr) + fclose(outputFile); + outputFile = newFile; // Relaunch thread nextInvocationIndexToBePersisted = 0; // Reset the dictionary diff --git a/runtime/RuntimeLogger.h b/runtime/RuntimeLogger.h index 65dcee0..c44ff0e 100644 --- a/runtime/RuntimeLogger.h +++ b/runtime/RuntimeLogger.h @@ -17,6 +17,7 @@ #define RUNTIME_NANOLOG_H #include +#include #include #include @@ -235,7 +236,7 @@ using namespace NanoLog; // File handle for the output file; should only be opened once at the // construction of the LogCompressor - int outputFd; + FILE* outputFile; // Buffer to stage compressed log message DoubleBuffer buffer; @@ -276,10 +277,6 @@ using namespace NanoLog; // Metric: Number of bytes written to the output file (includes padding) uint64_t totalBytesWritten; - // Metric: Number of pad bytes written to round the file to the nearest - // 512B - uint64_t padBytesWritten; - // Metric: Number of log statements compressed and outputted. uint64_t logsProcessed;