From 8737b825eaa01a9e2d23b7ee3256f3cf82f66130 Mon Sep 17 00:00:00 2001 From: swiftraccoon Date: Mon, 13 Nov 2023 23:54:31 -0500 Subject: [PATCH] Refactor debug flags and add debugging output to curlHelper.cpp - Added boolean flags for debugging different components. - Added debugging output to functions in curlHelper.cpp. - Refactored code to use the new debug flags to control the output of debugging messages. --- include/ConfigSingleton.h | 10 +++++ sample-config.yaml | 17 +++++++- src/ConfigSingleton.cpp | 12 +++++- src/curlHelper.cpp | 75 ++++++++++++++++++++++++++-------- src/fileProcessor.cpp | 28 +++++++++---- src/main.cpp | 15 +++++-- src/transcriptionProcessor.cpp | 19 +++++++-- 7 files changed, 141 insertions(+), 35 deletions(-) diff --git a/include/ConfigSingleton.h b/include/ConfigSingleton.h index 1d618e8..042ccac 100644 --- a/include/ConfigSingleton.h +++ b/include/ConfigSingleton.h @@ -28,6 +28,11 @@ class ConfigSingleton int getErrorWindowSeconds() const; int getRateLimitWindowSeconds() const; int getMinDurationSeconds() const; + bool isDebugCurlHelper() const; + bool isDebugDatabaseManager() const; + bool isDebugFileProcessor() const; + bool isDebugMain() const; + bool isDebugTranscriptionProcessor() const; private: ConfigSingleton() = default; @@ -50,6 +55,11 @@ class ConfigSingleton int errorWindowSeconds; int rateLimitWindowSeconds; int minDurationSeconds; + bool debugCurlHelper; + bool debugDatabaseManager; + bool debugFileProcessor; + bool debugMain; + bool debugTranscriptionProcessor; }; #endif // CONFIG_SINGLETON_H diff --git a/sample-config.yaml b/sample-config.yaml index cd7da0d..f0ae236 100644 --- a/sample-config.yaml +++ b/sample-config.yaml @@ -4,7 +4,8 @@ # used in main.cpp DirectoryToMonitor: "/usr/src/app/cpp-sdrtrunk-transcriber/" -# Number of seconds to wait before looping +# Number of seconds we should wait before we check the directory after processing +# the files that were in it. This is to prevent the program from using too much CPU. # used in main.cpp LoopWaitSeconds: 60 @@ -58,4 +59,16 @@ NCSHP_CALLSIGNS_FILE: "/usr/src/app/cpp-sdrtrunk-transcriber/howToFormatYourJSON # JSON object files for signals SIGNALS_FILE: "/usr/src/app/cpp-sdrtrunk-transcriber/howToFormatYourJSON.json" -NCSHP_SIGNALS_FILE: "/usr/src/app/cpp-sdrtrunk-transcriber/howToFormatYourJSON.json" \ No newline at end of file +NCSHP_SIGNALS_FILE: "/usr/src/app/cpp-sdrtrunk-transcriber/howToFormatYourJSON.json" + + +############# +# DEBUGGING # +############# +# If you want to see verbose output for a specific component, set it to true here. +# Warning: This will generate a lot of output! +DEBUG_CURL_HELPER: false +DEBUG_DATABASE_MANAGER: false +DEBUG_FILE_PROCESSOR: false +DEBUG_MAIN: false +DEBUG_TRANSCRIPTION_PROCESSOR: false \ No newline at end of file diff --git a/src/ConfigSingleton.cpp b/src/ConfigSingleton.cpp index 2fe3df1..86057db 100644 --- a/src/ConfigSingleton.cpp +++ b/src/ConfigSingleton.cpp @@ -24,6 +24,11 @@ void ConfigSingleton::initialize(const YAML::Node &config) errorWindowSeconds = config["ERROR_WINDOW_SECONDS"].as(); rateLimitWindowSeconds = config["RATE_LIMIT_WINDOW_SECONDS"].as(); minDurationSeconds = config["MIN_DURATION_SECONDS"].as(); + debugCurlHelper = config["DEBUG_CURL_HELPER"].as(false); + debugDatabaseManager = config["DEBUG_DATABASE_MANAGER"].as(false); + debugFileProcessor = config["DEBUG_FILE_PROCESSOR"].as(false); + debugMain = config["DEBUG_MAIN"].as(false); + debugTranscriptionProcessor = config["DEBUG_TRANSCRIPTION_PROCESSOR"].as(false); } std::string ConfigSingleton::getTensignFile() const { return tensignFile; } @@ -40,4 +45,9 @@ int ConfigSingleton::getMinDurationSeconds() const { return minDurationSeconds; int ConfigSingleton::getMaxRetries() const { return maxRetries; } int ConfigSingleton::getMaxRequestsPerMinute() const { return maxRequestsPerMinute; } int ConfigSingleton::getErrorWindowSeconds() const { return errorWindowSeconds; } -int ConfigSingleton::getRateLimitWindowSeconds() const { return rateLimitWindowSeconds; } \ No newline at end of file +int ConfigSingleton::getRateLimitWindowSeconds() const { return rateLimitWindowSeconds; } +bool ConfigSingleton::isDebugCurlHelper() const { return debugCurlHelper; } +bool ConfigSingleton::isDebugDatabaseManager() const { return debugDatabaseManager; } +bool ConfigSingleton::isDebugFileProcessor() const { return debugFileProcessor; } +bool ConfigSingleton::isDebugMain() const { return debugMain; } +bool ConfigSingleton::isDebugTranscriptionProcessor() const { return debugTranscriptionProcessor; } diff --git a/src/curlHelper.cpp b/src/curlHelper.cpp index e58413d..32760ce 100644 --- a/src/curlHelper.cpp +++ b/src/curlHelper.cpp @@ -109,7 +109,10 @@ bool containsApiError(const std::string &response) // Check file existence and readability void checkFileValidity(const std::string &file_path) { - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp checkFileValidity Checking if file exists." << std::endl; + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp checkFileValidity Checking if file exists." << std::endl; + } if (!std::filesystem::exists(file_path)) { throw std::runtime_error("[" + getCurrentTime() + "]" + " curlHelper.cpp checkFileValidity File does not exist: " + file_path); @@ -117,7 +120,10 @@ void checkFileValidity(const std::string &file_path) } std::ifstream file(file_path); - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp checkFileValidity Checking if file is readable." << std::endl; + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp checkFileValidity Checking if file is readable." << std::endl; + } if (!file.good()) { throw std::runtime_error("[" + getCurrentTime() + "]" + " curlHelper.cpp checkFileValidity Cannot read file: " + file_path); @@ -131,9 +137,15 @@ void handleRateLimiting() { std::chrono::seconds rateLimitWindow(config.getRateLimitWindowSeconds()); int maxRequestsPerMinute = config.getMaxRequestsPerMinute(); - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp Entered handleRateLimiting" << std::endl; + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp Entered handleRateLimiting" << std::endl; + } auto now = std::chrono::steady_clock::now(); - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp handleRateLimiting rateLimitWindow: " << rateLimitWindow.count() << " seconds" << std::endl; + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp handleRateLimiting rateLimitWindow: " << rateLimitWindow.count() << " seconds" << std::endl; + } // Remove timestamps outside the 1-minute window while (!requestTimestamps.empty() && (now - requestTimestamps.front() > rateLimitWindow)) { @@ -146,11 +158,17 @@ void handleRateLimiting() } // Check if we've exceeded the rate limit - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp handleRateLimiting maxRequestsPerMinute: " << maxRequestsPerMinute << std::endl; + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp handleRateLimiting maxRequestsPerMinute: " << maxRequestsPerMinute << std::endl; + } if (requestTimestamps.size() >= maxRequestsPerMinute) { auto sleep_duration = rateLimitWindow - (now - requestTimestamps.front()); - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp handleRateLimiting Rate limit reached, sleeping for " << sleep_duration.count() << " seconds." << std::endl; + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp handleRateLimiting Rate limit reached, sleeping for " << sleep_duration.count() << " seconds." << std::endl; + } std::this_thread::sleep_for(sleep_duration); } } @@ -161,18 +179,29 @@ std::string curl_transcribe_audio(const std::string &file_path, const std::strin int maxRetries = config.getMaxRetries(); int maxRequestsPerMinute = config.getMaxRequestsPerMinute(); // std::chrono::seconds errorWindow(config.getErrorWindowSeconds()); - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio called with file path: " << file_path << std::endl; + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio called with file path: " << file_path << std::endl; + } checkFileValidity(file_path); - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio Entering retry loop." << std::endl; - std::cout.flush(); + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio Entering retry loop." << std::endl; + } try { - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio maxRetries: " << maxRetries << std::endl; - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio maxRequestsPerMinute: " << maxRequestsPerMinute << std::endl; + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio maxRetries: " << maxRetries << std::endl; + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio maxRequestsPerMinute: " << maxRequestsPerMinute << std::endl; + } for (int retryCount = 0; retryCount < maxRetries; ++retryCount) { - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio attempt " << (retryCount + 1) << " of " << maxRetries << std::endl; - std::cout.flush(); + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio retryCount: " << retryCount << std::endl; + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio attempt " << (retryCount + 1) << " of " << maxRetries << std::endl; + } handleRateLimiting(); CURL *curl = curl_easy_init(); @@ -189,17 +218,25 @@ std::string curl_transcribe_audio(const std::string &file_path, const std::strin curl_easy_setopt(curl, CURLOPT_URL, API_URL.c_str()); curl_easy_setopt(curl, CURLOPT_MIMEPOST, mime); - - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio Making API call." << std::endl; + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio Making API call." << std::endl; + } std::string response = makeCurlRequest(curl, mime); - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio Received response: " << response << std::endl; + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio Received response: " << response << std::endl; + } curl_easy_cleanup(curl); curl_mime_free(mime); if (!containsApiError(response) && isValidResponse(response)) { - std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio Valid response received." << std::endl; + if (ConfigSingleton::getInstance().isDebugCurlHelper()) + { + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio Valid response received." << std::endl; + } return response; // Success, return the response } @@ -224,6 +261,8 @@ std::string curl_transcribe_audio(const std::string &file_path, const std::strin std::cerr << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio Majority of retries failed due to API errors." << std::endl; exit(EXIT_FAILURE); } - + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio We exited the TRY block. investigate." << std::endl; + std::cout << "[" << getCurrentTime() << "] curlHelper.cpp curl_transcribe_audio We were on this file: " << file_path << std::endl; + exit(EXIT_FAILURE); // return "curlHelper.cpp_UNABLE_TO_TRANSCRIBE_CHECK_FILE"; } \ No newline at end of file diff --git a/src/fileProcessor.cpp b/src/fileProcessor.cpp index d7d21bc..cd30b55 100644 --- a/src/fileProcessor.cpp +++ b/src/fileProcessor.cpp @@ -218,8 +218,11 @@ void moveFiles(const FileData &fileData, const std::string &directoryToMonitor) { std::filesystem::create_directory(subDir); } - - // 22 std::cout << "[" << getCurrentTime() << "] " << "fileProcessor.cpp moveFiles Moving file from: " << fileData.filepath << " to: " << (subDir / fileData.filename) << std::endl; // Debug statement + if (ConfigSingleton::getInstance().isDebugFileProcessor()) + { + std::cout << "[" << getCurrentTime() << "] " + << "fileProcessor.cpp moveFiles Moving file from: " << fileData.filepath << " to: " << (subDir / fileData.filename) << std::endl; + } if (std::filesystem::exists(fileData.filepath)) { @@ -227,8 +230,11 @@ void moveFiles(const FileData &fileData, const std::string &directoryToMonitor) } std::string txt_filename = fileData.filename.substr(0, fileData.filename.size() - 4) + ".txt"; - - // 22 std::cout << "[" << getCurrentTime() << "] " << "fileProcessor.cpp moveFiles Moving txt from: " << txt_filename << " to: " << (subDir / txt_filename) << std::endl; // Debug statement + if (ConfigSingleton::getInstance().isDebugFileProcessor()) + { + std::cout << "[" << getCurrentTime() << "] " + << "fileProcessor.cpp moveFiles Moving txt from: " << txt_filename << " to: " << (subDir / txt_filename) << std::endl; + } if (std::filesystem::exists(txt_filename)) { @@ -243,11 +249,19 @@ FileData processFile(const std::filesystem::path &path, const std::string &direc { FileData fileData; std::string file_path = path.string(); - // 22 std::cout << "[" << getCurrentTime() << "] " << "fileProcessor.cpp processFile Processing file: " << file_path << std::endl; // Debug statement + if (ConfigSingleton::getInstance().isDebugFileProcessor()) + { + std::cout << "[" << getCurrentTime() << "] " + << "fileProcessor.cpp processFile Processing file: " << file_path << std::endl; + } bool shouldSkip = skipFile(file_path); float duration = validateDuration(file_path, fileData); - // 22 std::cout << "[" << getCurrentTime() << "] " << "fileProcessor.cpp processFile Should skip: " << shouldSkip << std::endl; // Debug statement - shouldSkip = shouldSkip || (duration == 0.0); // Update this line + if (ConfigSingleton::getInstance().isDebugFileProcessor()) + { + std::cout << "[" << getCurrentTime() << "] " + << "fileProcessor.cpp processFile Should skip: " << shouldSkip << std::endl; + } + shouldSkip = shouldSkip || (duration == 0.0); if (shouldSkip) { return FileData(); // Skip further processing diff --git a/src/main.cpp b/src/main.cpp index 9ee3378..de62812 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -62,15 +62,24 @@ void processDirectory(const std::string &directoryToMonitor, const YAML::Node &c { if (entry.path().extension() == MP3_EXTENSION) { - // 22 std::cout << "[" << getCurrentTime() << "] " << "main.cpp processDirectory Processing directory: " << directoryToMonitor << std::endl; - // 22 std::cout << "[" << getCurrentTime() << "] " << "main.cpp processDirectory Checking file: " << entry.path() << std::endl; + if (ConfigSingleton::getInstance().isDebugMain()) + { + std::cout << "[" << getCurrentTime() << "] " + << "main.cpp processDirectory Processing directory: " << directoryToMonitor << std::endl; + std::cout << "[" << getCurrentTime() << "] " + << "main.cpp processDirectory Checking file: " << entry.path() << std::endl; + } try { fileData = processFile(entry.path(), directoryToMonitor, OPENAI_API_KEY); } catch (const std::runtime_error &e) { - // 22 std::cerr << "[" << getCurrentTime() << "] " << "main.cpp processDirectory Skipping file: " << e.what() << std::endl; + if (ConfigSingleton::getInstance().isDebugMain()) + { + std::cerr << "[" << getCurrentTime() << "] " + << "main.cpp processDirectory Skipping file: " << e.what() << std::endl; + } continue; // Move on to the next file } diff --git a/src/transcriptionProcessor.cpp b/src/transcriptionProcessor.cpp index 833a91e..44194d6 100644 --- a/src/transcriptionProcessor.cpp +++ b/src/transcriptionProcessor.cpp @@ -86,7 +86,11 @@ std::unordered_map readMappingFile(const std::string & std::ifstream file(filePath); if (!file.is_open()) { - // 22 std::cerr << "[" << getCurrentTime() << "] " << "transcriptionProcessor.cpp readMappingFile Could not open file: " << filePath << std::endl; + if (ConfigSingleton::getInstance().isDebugTranscriptionProcessor()) + { + std::cerr << "[" << getCurrentTime() << "] " + << "transcriptionProcessor.cpp readMappingFile Could not open file: " << filePath << std::endl; + } return mapping; } @@ -94,8 +98,11 @@ std::unordered_map readMappingFile(const std::string & { nlohmann::json j; file >> j; - // 22 std::cerr << "[" << getCurrentTime() << "] " - // 22 << "transcriptionProcessor.cpp readMappingFile JSON: " << j << std::endl; + if (ConfigSingleton::getInstance().isDebugTranscriptionProcessor()) + { + std::cerr << "[" << getCurrentTime() << "] " + << "transcriptionProcessor.cpp readMappingFile JSON: " << j << std::endl; + } for (const auto &[key, value] : j.items()) { if (value.is_string()) @@ -146,7 +153,11 @@ std::unordered_map readMappingFile(const std::string & std::string extractActualTranscription(const std::string &transcription) { - // 22 std::cerr << "[" << getCurrentTime() << "] " << "transcriptionProcessor.cpp extractActualTranscription Received transcription string: " << transcription << std::endl; + if (ConfigSingleton::getInstance().isDebugTranscriptionProcessor()) + { + std::cerr << "[" << getCurrentTime() << "] " + << "transcriptionProcessor.cpp extractActualTranscription Received transcription string: " << transcription << std::endl; + } static const std::regex text_regex("\"text\":\"([^\"]+)\""); std::smatch match; return std::regex_search(transcription, match, text_regex) && match.size() > 1 ? match.str(1) : "";