From 641d643a246e49641cb3b82cf770e003a015bff8 Mon Sep 17 00:00:00 2001 From: Annapurna Date: Wed, 30 Jan 2013 12:38:25 +0530 Subject: [PATCH 1/7] _wrote_num_messgaes and received good mismatch --- src/store.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/store.cpp b/src/store.cpp index efa48a62..693a83d6 100644 --- a/src/store.cpp +++ b/src/store.cpp @@ -559,9 +559,6 @@ void FileStoreBase::printStats() { string log_str = msg.str(); LOG_OPER("[%s]", log_str.c_str()); /*stats_file->write(msg.str());*/ - // update the stats - g_Handler->incCounter(categoryHandled, fsType + "_wrote_num_messages", eventsWritten); - g_Handler->incCounter(categoryHandled, fsType + "_wrote_bytes", currentSize); stats_file->close(); } @@ -744,6 +741,10 @@ bool FileStore::openInternal(bool incrementFilename, struct tm* current_time) { } // else it confuses the filename code on reads + // update stats + g_Handler->incCounter(categoryHandled, fsType + "_wrote_num_messages", eventsWritten); + g_Handler->incCounter(categoryHandled, fsType + "_wrote_bytes", currentSize); + LOG_OPER("[%s] Opened file <%s> for writing", categoryHandled.c_str(), file.c_str()); From cf4cb584d5ab2b752bb09873c64f1e12e776f8a3 Mon Sep 17 00:00:00 2001 From: Annapurna Date: Wed, 6 Feb 2013 12:55:52 +0530 Subject: [PATCH 2/7] logging done on close of file not on rotation --- src/store.cpp | 30 ++++++++++++++++++++++++++---- src/store.h | 1 + 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/store.cpp b/src/store.cpp index 693a83d6..0e1974ec 100644 --- a/src/store.cpp +++ b/src/store.cpp @@ -542,7 +542,7 @@ void FileStoreBase::printStats() { return; } - time_t rawtime = time(NULL); + /*time_t rawtime = time(NULL); struct tm timeinfo; localtime_r(&rawtime, &timeinfo); @@ -558,7 +558,7 @@ void FileStoreBase::printStats() { string log_str = msg.str(); LOG_OPER("[%s]", log_str.c_str()); - /*stats_file->write(msg.str());*/ + stats_file->write(msg.str());*/ stats_file->close(); } @@ -695,7 +695,7 @@ bool FileStore::openInternal(bool incrementFilename, struct tm* current_time) { if (writeMeta) { writeFile->write(meta_logfile_prefix + file); } - writeFile->close(); + closeWriteFile(); } writeFile = FileInterface::createFileInterface(fsType, file, isBufferFile); @@ -769,12 +769,34 @@ bool FileStore::isOpen() { return writeFile && writeFile->isOpen(); } -void FileStore::close() { +void FileStroe::closeWriteFile() { if (writeFile) { writeFile->close(); + if (writeStats && !(writeFile->isOpen()) ) { + time_t rawtime = time(NULL); + struct tm timeinfo; + localtime_r(&rawtime, &timeinfo); + + ostringstream msg; + msg << timeinfo.tm_year + 1900 << '-' + << setw(2) << setfill('0') << timeinfo.tm_mon + 1 << '-' + << setw(2) << setfill('0') << timeinfo.tm_mday << '-' + << setw(2) << setfill('0') << timeinfo.tm_hour << ':' + << setw(2) << setfill('0') << timeinfo.tm_min; + + msg << " wrote <" << currentSize << "> bytes in <" << eventsWritten + << "> events to file <" << currentFilename << ">" << endl; + + string log_str = msg.str(); + LOG_OPER("[%s]", log_str.c_str()); + } } } +void FileStore::close() { + closeWriteFile(); +} + void FileStore::flush() { if (writeFile) { writeFile->flush(); diff --git a/src/store.h b/src/store.h index c64a0408..bda9b080 100644 --- a/src/store.h +++ b/src/store.h @@ -200,6 +200,7 @@ class FileStore : public FileStoreBase { bool handleMessages(boost::shared_ptr messages); bool isOpen(); void configure(pStoreConf configuration, pStoreConf parent); + void closeWriteFile(); void close(); void flush(); From 452a8a811d46c815f9d2365ad4626ce42b492399 Mon Sep 17 00:00:00 2001 From: Annapurna Date: Wed, 6 Feb 2013 15:34:02 +0530 Subject: [PATCH 3/7] Typo modified --- src/store.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/store.cpp b/src/store.cpp index 0e1974ec..e6ace644 100644 --- a/src/store.cpp +++ b/src/store.cpp @@ -769,7 +769,7 @@ bool FileStore::isOpen() { return writeFile && writeFile->isOpen(); } -void FileStroe::closeWriteFile() { +void FileStore::closeWriteFile() { if (writeFile) { writeFile->close(); if (writeStats && !(writeFile->isOpen()) ) { From 6a2c7909b3637655e232f60445badd078fa63b50 Mon Sep 17 00:00:00 2001 From: Annapurna Date: Fri, 8 Feb 2013 13:01:57 +0530 Subject: [PATCH 4/7] eventsWritten incremented on every msg written --- src/store.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/store.cpp b/src/store.cpp index e6ace644..7c268eed 100644 --- a/src/store.cpp +++ b/src/store.cpp @@ -924,6 +924,7 @@ bool FileStore::writeMessages(boost::shared_ptr messages, num_written += num_buffered; currentSize += current_size_buffered; + eventsWritten += num_buffered; num_buffered = 0; current_size_buffered = 0; write_buffer = ""; @@ -941,8 +942,6 @@ bool FileStore::writeMessages(boost::shared_ptr messages, success = false; } - eventsWritten += num_written; - if (!success) { close(); From 6d75029e2c77d7e9b3c4b3c0deb1b9326aca490e Mon Sep 17 00:00:00 2001 From: Annapurna Date: Mon, 18 Feb 2013 18:04:22 +0530 Subject: [PATCH 5/7] counters incremented in closeWriteFile() --- src/store.cpp | 15 +++++++++------ src/store.h | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/store.cpp b/src/store.cpp index 7c268eed..29800ae0 100644 --- a/src/store.cpp +++ b/src/store.cpp @@ -201,6 +201,7 @@ FileStoreBase::FileStoreBase(StoreQueue* storeq, writeStats(false), rotateOnReopen(false), currentSize(0), + eventSize(0), lastRollTime(0), eventsWritten(0) { } @@ -740,17 +741,12 @@ bool FileStore::openInternal(bool incrementFilename, struct tm* current_time) { writeFile->createSymlink(symtarget, symlinkName); } // else it confuses the filename code on reads - - // update stats - g_Handler->incCounter(categoryHandled, fsType + "_wrote_num_messages", eventsWritten); - g_Handler->incCounter(categoryHandled, fsType + "_wrote_bytes", currentSize); LOG_OPER("[%s] Opened file <%s> for writing", categoryHandled.c_str(), file.c_str()); currentSize = writeFile->fileSize(); currentFilename = file; - eventsWritten = 0; setStatus(""); } @@ -784,12 +780,18 @@ void FileStore::closeWriteFile() { << setw(2) << setfill('0') << timeinfo.tm_hour << ':' << setw(2) << setfill('0') << timeinfo.tm_min; - msg << " wrote <" << currentSize << "> bytes in <" << eventsWritten + msg << " wrote <" << eventSize << "> bytes in <" << eventsWritten << "> events to file <" << currentFilename << ">" << endl; string log_str = msg.str(); LOG_OPER("[%s]", log_str.c_str()); } + // update stats + g_Handler->incCounter(categoryHandled, fsType + "_wrote_num_messages", eventsWritten); + g_Handler->incCounter(categoryHandled, fsType + "_wrote_bytes", eventSize); + + eventsWritten = 0; + eventsSize = 0; } } @@ -924,6 +926,7 @@ bool FileStore::writeMessages(boost::shared_ptr messages, num_written += num_buffered; currentSize += current_size_buffered; + eventSize += current_size_buffered; eventsWritten += num_buffered; num_buffered = 0; current_size_buffered = 0; diff --git a/src/store.h b/src/store.h index bda9b080..7c2aef24 100644 --- a/src/store.h +++ b/src/store.h @@ -178,6 +178,7 @@ class FileStoreBase : public Store { unsigned long eventsWritten; // This is how many events this process has // written to the currently open file. It is NOT // necessarily the number of lines in the file + unsigned long eventSize; // size corresponding to eventsWritten private: // disallow copy, assignment, and empty construction From f2591f6bc9a1f7a8ae6e9dc1f4314f55dcc58f8a Mon Sep 17 00:00:00 2001 From: Annapurna Date: Mon, 18 Feb 2013 18:18:52 +0530 Subject: [PATCH 6/7] typo modified --- src/store.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/store.cpp b/src/store.cpp index 29800ae0..efd76378 100644 --- a/src/store.cpp +++ b/src/store.cpp @@ -791,7 +791,7 @@ void FileStore::closeWriteFile() { g_Handler->incCounter(categoryHandled, fsType + "_wrote_bytes", eventSize); eventsWritten = 0; - eventsSize = 0; + eventSize = 0; } } From beedd5aedbf4b7486ce639a0c9445364c53769e6 Mon Sep 17 00:00:00 2001 From: Annapurna Date: Tue, 19 Feb 2013 12:31:42 +0530 Subject: [PATCH 7/7] removed message from printStats() --- src/store.cpp | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/src/store.cpp b/src/store.cpp index efd76378..16eebfa1 100644 --- a/src/store.cpp +++ b/src/store.cpp @@ -528,7 +528,7 @@ void FileStoreBase::printStats() { if (!writeStats) { return; } - + string filename(filePath); filename += "/scribe_stats"; @@ -541,25 +541,10 @@ void FileStoreBase::printStats() { categoryHandled.c_str(), filename.c_str(), fsType.c_str()); // This isn't enough of a problem to change our status return; - } - - /*time_t rawtime = time(NULL); - struct tm timeinfo; - localtime_r(&rawtime, &timeinfo); - - ostringstream msg; - msg << timeinfo.tm_year + 1900 << '-' - << setw(2) << setfill('0') << timeinfo.tm_mon + 1 << '-' - << setw(2) << setfill('0') << timeinfo.tm_mday << '-' - << setw(2) << setfill('0') << timeinfo.tm_hour << ':' - << setw(2) << setfill('0') << timeinfo.tm_min; - - msg << " wrote <" << currentSize << "> bytes in <" << eventsWritten - << "> events to file <" << currentFilename << ">" << endl; - - string log_str = msg.str(); - LOG_OPER("[%s]", log_str.c_str()); - stats_file->write(msg.str());*/ + } + + //scribe_stats should be created and message written if/when hdfs-append works + //stats_file->write(msg.str()); stats_file->close(); } @@ -785,6 +770,9 @@ void FileStore::closeWriteFile() { string log_str = msg.str(); LOG_OPER("[%s]", log_str.c_str()); + + // the message that is logged here should be moved to printStats() to be + // appended to scribe_stats when hdfs append works. } // update stats g_Handler->incCounter(categoryHandled, fsType + "_wrote_num_messages", eventsWritten);