2014 network limit 1.3 fix log/path/data +utils

+toc -doc -drmonero

Fixed the windows path, and improved logging and data
(for graph) logging, fixed some locks and added more checks.

Still there is a locking error,
not added by my patches, but present in master version
(locking of map/list of peers).

.gitignore

@@ -59,7 +59,6 @@ version/
 ### CMake ###
 CMakeCache.txt
 CMakeFiles
-Makefile
 cmake_install.cmake
 install_manifest.txt
 *.cmake

contrib/otshell_utils/utils.cpp

@@ -160,7 +160,7 @@ std::unique_ptr<T> make_unique( Args&& ...args )
 #endif
 // ====================================================================
 
-char cFilesystemUtils::GetDirSeparator() {
+char cFilesystemUtils::GetDirSeparatorSys() {
 	// TODO nicer os detection?
 	#if defined(OS_TYPE_POSIX)
 		return '/';
@@ -171,51 +171,74 @@ char cFilesystemUtils::GetDirSeparator() {
 	#endif
 }
 
+char cFilesystemUtils::GetDirSeparatorInter() {
+	return '/';
+}
+
+string cFilesystemUtils::FileInternalToSystem(const std::string &name) {
+	string ret;
+	ret.resize(name.size());
+	std::replace_copy(name.begin(), name.end(), ret.begin(), 
+		GetDirSeparatorInter() , GetDirSeparatorSys());
+	return ret;
+}
+
+string cFilesystemUtils::FileSystemToInternal(const std::string &name) {
+	string ret;
+	ret.reserve(name.size());
+	std::replace_copy(name.begin(), name.end(), ret.begin(), 
+		GetDirSeparatorSys() , GetDirSeparatorInter());
+	return ret;
+}
+
 bool cFilesystemUtils::CreateDirTree(const std::string & dir, bool only_below) {
 	const bool dbg=false;
 	//struct stat st;
-	const char dirch = cFilesystemUtils::GetDirSeparator();
+	const char dirchS = cFilesystemUtils::GetDirSeparatorSys();
+	const char dirchI = cFilesystemUtils::GetDirSeparatorInter();
 	std::istringstream iss(dir);
-	string part, sofar="";
+	string partI; // current par is in internal format (though it should not matter since it doesn't contain any slashes). eg "bar"
+	string sofarS=""; // sofarS - the so far created dir part is in SYSTEM format. eg "foo/bar"
 	if (dir.size()<1) return false; // illegal name
 	// dir[0] is valid from here
-	if  (only_below && (dir[0]==dirch)) return false; // no jumping to top (on any os)
-	while (getline(iss,part,dirch)) {
-		if (dbg) cout << '['<<part<<']' << endl;
-		sofar += part;
-		if (part.size()<1) return false; // bad format?
-		if ((only_below) && (part=="..")) return false; // going up
-
-		if (dbg) cout << "test ["<<sofar<<"]"<<endl;
+	if  ( only_below  &&  ((dir[0]==dirchS) || (dir[0]==dirchI))) return false; // no jumping to top (on any os)
+
+	while (getline(iss,partI,dirchI)) { // get new component eg "bar" into part
+		if (dbg) cout << '['<<partI<<']' << endl;
+		sofarS += partI;
+		if (partI.size()<1) return false; // bad format?
+		if ((only_below) && (partI=="..")) return false; // trying to go up
+
+		if (dbg) cout << "test ["<<sofarS<<"]"<<endl;
 		// TODO nicer os detection?
 		#if defined(OS_TYPE_POSIX)
 			struct stat st;
-			bool exists = stat(sofar.c_str() ,&st) == 0; // *
+			bool exists = stat(sofarS.c_str() ,&st) == 0; // *
 			if (exists) {
 				if (! S_ISDIR(st.st_mode)) {
 					// std::cerr << "This exists, but as a file: [" << sofar << "]" << (size_t)st.st_ino << endl;
 					return false; // exists but is a file nor dir
 				}
 			}
 		#elif defined(OS_TYPE_WINDOWS)
-			DWORD dwAttrib = GetFileAttributesA(sofar.c_str());
+			DWORD dwAttrib = GetFileAttributesA(sofarS.c_str());
 			bool exists = (dwAttrib != INVALID_FILE_ATTRIBUTES && (dwAttrib & FILE_ATTRIBUTE_DIRECTORY));
 		#else
 			#error "Do not know how to compile this for your platform."
 		#endif
 
 		if (!exists) {
-			if (dbg) cout << "mkdir ["<<sofar<<"]"<<endl;
+			if (dbg) cout << "mkdir ["<<sofarS<<"]"<<endl;
 			#if defined(OS_TYPE_POSIX)
-				bool ok = 0==  mkdir(sofar.c_str(), 0700); // ***
+				bool ok = 0==  mkdir(sofarS.c_str(), 0700); // ***
 			#elif defined(OS_TYPE_WINDOWS)
-				bool ok = (bool) CreateDirectoryA(sofar.c_str(), NULL); // TODO use -W() after conversion to unicode UTF16
+				bool ok = (bool) CreateDirectoryA(sofarS.c_str(), NULL); // TODO use -W() after conversion to unicode UTF16
 			#else
 				#error "Do not know how to compile this for your platform."
 			#endif
 			if (!ok) return false;
 		}
-		sofar += cFilesystemUtils::GetDirSeparator();
+		sofarS += dirchS;
 	}
 	return true;
 }
@@ -225,7 +248,7 @@ namespace nDetail {
 
 cDebugScopeGuard::cDebugScopeGuard() : mLevel(-1) {
 }
-
+	
 cDebugScopeGuard::~cDebugScopeGuard() {
 	if (mLevel != -1) {
 		gCurrentLogger.write_stream(mLevel,mChan) << mMsg << " ... end" << gCurrentLogger.endline() << std::flush;
@@ -244,11 +267,15 @@ void cDebugScopeGuard::Assign(const string &chan, const int level, const string
 
 cLogger::cLogger() :
 mStream(NULL),
+mStreamBrokenDebug(NULL),
+mIsBroken(true), // before constructor finishes
 mLevel(85),
 mThread2Number_Biggest(0) // the CURRENT biggest value (no thread yet in map)
 {
 	mStream = & std::cout;
+	mStreamBrokenDebug = & std::cerr;
 	Thread2Number( std::this_thread::get_id() ); // convert current id to short number, useful to reserve a number so that main thread is usually called 1
+	mIsBroken=false; // ok, constr. succeeded, so string is not broken now
 }
 
 cLogger::~cLogger() {
@@ -259,12 +286,28 @@ cLogger::~cLogger() {
 	}
 }
 
+void cLogger::SetStreamBroken() {
+	SetStreamBroken("(no additional details about this problem)");
+}
+
+void cLogger::SetStreamBroken(const std::string &msg) {
+	if (!mIsBroken) { // if not already marked as broken
+		std::cerr << OT_CODE_STAMP << "WARNING: due to debug stream problem ("<<msg<<") - switching back to fallback stream (e.g. cerr)" << std::endl;
+		if (mStreamBrokenDebug == nullptr) {
+			std::cerr << OT_CODE_STAMP << " ERROR: in addition, while reporting this problem, mStreamBrokenDebug stream is NULL." << std::endl;
+		} else {
+			(*mStreamBrokenDebug) << OT_CODE_STAMP << "WARNING: due to debug stream problem ("<<msg<<") - switching back to fallback stream (e.g. cerr)" << std::endl;
+		}
+		mIsBroken = true;
+	}
+}
+
 std::ostream & cLogger::write_stream(int level) {
 	return write_stream(level,"");
 }
 
 std::ostream & cLogger::write_stream(int level, const std::string & channel ) {
-	if ((level >= mLevel) && (mStream)) {
+	if ((level >= mLevel) && (mStream)) { // TODO now disabling mStream also disables writting to any channel
 		ostream & output = SelectOutput(level,channel);
 		output << icon(level) << ' ';
 		std::thread::id this_id = std::this_thread::get_id();
@@ -278,33 +321,76 @@ std::string cLogger::GetLogBaseDir() const {
 	return "log";
 }
 
-void cLogger::OpenNewChannel(const std::string & channel) {
-	size_t last_split = channel.find_last_of(cFilesystemUtils::GetDirSeparator());
-	// log/test/aaa
-	//         ^----- last_split
-	string dir = GetLogBaseDir() + cFilesystemUtils::GetDirSeparator() + channel.substr(0, last_split);
-	string basefile = channel.substr(last_split+1) + ".log";
-	string fname = dir + cFilesystemUtils::GetDirSeparator() + cFilesystemUtils::GetDirSeparator() + basefile;
-	_dbg1("Starting debug to channel file: " + fname + " in directory ["+dir+"]");
-	bool dirok = cFilesystemUtils::CreateDirTree(dir);
-	if (!dirok) { const string msg = "In logger failed to open directory (" + dir +")."; _erro(msg); throw std::runtime_error(msg); }
-	std::ofstream * thefile = new std::ofstream( fname.c_str() );
-	*thefile << "====== (Log opened: " << fname << ") ======" << endl;
-	mChannels.insert( std::pair<string,std::ofstream*>(channel , thefile ) );
-}
-
-std::ostream & cLogger::SelectOutput(int level, const std::string & channel) {
-	if (channel=="") return *mStream;
-	auto obj = mChannels.find(channel);
-	if (obj == mChannels.end()) { // new channel
-		OpenNewChannel(channel);
-		return SelectOutput(level,channel);
+void cLogger::OpenNewChannel(const std::string & channel) noexcept {
+	try {
+		std::cerr<<"openning channel for channel="<<channel<<endl;
+		OpenNewChannel_(channel);
+	}
+	catch (const std::exception &except) {
+		SetStreamBroken(OT_CODE_STAMP + " Got exception when opening debug channel: " + ToStr(except.what()));
+	}
+	catch (...) {
+		SetStreamBroken(OT_CODE_STAMP + " Got not-standard exception when opening debug channel.");
+	}
+}
+
+void cLogger::OpenNewChannel_(const std::string & channel) { // channel=="net/sleep"
+	size_t last_split = channel.find_last_of(cFilesystemUtils::GetDirSeparatorInter());
+
+	string fname_system; // the full file name in system format
+
+	if (last_split==string::npos) { // The channel name has no directory, eg channel=="test"
+		string dir = GetLogBaseDir();
+		string basefile = channel + ".log";
+		string fname = dir + cFilesystemUtils::GetDirSeparatorInter() + basefile;
+		fname_system = cFilesystemUtils::FileInternalToSystem(fname); // <-
 	}
-	else { // existing
-		return * obj->second;
+	else { // there is a directory eg channel=="net/sleep"
+		// net/sleep 
+		//    ^----- last_split	
+		string dir = GetLogBaseDir() + cFilesystemUtils::GetDirSeparatorInter() + channel.substr(0, last_split);
+		string basefile = channel.substr(last_split+1) + ".log";
+		string fname = dir + cFilesystemUtils::GetDirSeparatorInter() + basefile;
+		fname_system = cFilesystemUtils::FileInternalToSystem(fname); // <-
+		bool dirok = cFilesystemUtils::CreateDirTree(dir);
+		if (!dirok) { string err = "In logger failed to open directory (" + dir +") for channel (" + channel +")"; throw std::runtime_error(err); }
 	}
+
+	std::ofstream * thefile = new std::ofstream( fname_system.c_str() ); // file system
+	*thefile << "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" << endl;
+    	cerr << "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" << endl;
+	mChannels.insert( std::pair<string,std::ofstream*>(channel , thefile ) ); // <- created the channel mapping
 }
 
+std::ostream & cLogger::SelectOutput(int level, const std::string & channel) noexcept {
+	try {
+		if (mIsBroken) return *mStreamBrokenDebug;
+		if (channel=="") return *mStream;
+
+		auto obj = mChannels.find(channel);
+		if (obj == mChannels.end()) { // not found - need to make new channel
+			OpenNewChannel(channel); // <- create channel
+			obj = mChannels.find(channel); // find again
+			if (obj == mChannels.end()) { // still not found! something is wrong
+				SetStreamBroken( OT_CODE_STAMP + " WARNING: can not get stream for channel="+ToStr(channel)+" level="+ToStr(channel) );
+				return *mStreamBrokenDebug;
+			}
+		}
+		auto the_stream_ptr = obj->second;
+		ASRT(the_stream_ptr);
+		return *the_stream_ptr; // <--- RETURN
+	} 
+	catch (std::exception &except) { 
+		SetStreamBroken( OT_CODE_STAMP + " Got exception: " + ToStr(except.what()) );
+		return *mStreamBrokenDebug;
+	}
+	catch (...) {
+		SetStreamBroken( OT_CODE_STAMP + " Got not-standard exception.");
+		return *mStreamBrokenDebug;
+	}
+
+	// dead code
+}
 
 void cLogger::setOutStreamFile(const string &fname) { // switch to using this file
 	_mark("WILL SWITCH DEBUG NOW to file: " << fname);

contrib/otshell_utils/utils.hpp

@@ -185,6 +185,7 @@ const char* DbgShortenCodeFileName(const char *s); ///< Returns a pointer to som
 /*** 
 @brief Class to write debug into. Used it by calling the debug macros _dbg1(...) _info(...) _erro(...) etc, NOT directly!
 @author rfree (maintainer) 
+@thread this class is NOT thread safe and must used only by one thread at once (use it via ot_debug_macros like _info macro they do proper locking)
 */
 class cLogger {
 	public:
@@ -201,15 +202,21 @@ class cLogger {
 		std::string endline() const; ///< returns string to be written at end of message
 
 	protected:
+		void SetStreamBroken(); ///< call in case of internal error in logger (e.g. can not open a file)
+		void SetStreamBroken(const std::string &msg); ///< same but with error message
+
 		unique_ptr<std::ofstream> mOutfile;
 		std::ostream * mStream; ///< pointing only! can point to our own mOutfile, or maye to global null stream
+		std::ostream * mStreamBrokenDebug; ///< pointing only! this is a pointer to some stream that should be used when normal debugging is broken eg std::cerr
+		bool mIsBroken; ///< is the debugging system broken (this should be set when internal problems occur and should cause fallback to std::cerr)
 
 		std::map< std::string , std::ofstream * > mChannels; // the ofstream objects are owned by this class
 
 		int mLevel; ///< current debug level
 
-		std::ostream & SelectOutput(int level, const std::string & channel);
-		void OpenNewChannel(const std::string & channel);
+		std::ostream & SelectOutput(int level, const std::string & channel) noexcept; ///< returns a proper stream for this level and channel (always usable string)
+		void OpenNewChannel(const std::string & channel) noexcept; ///< tries to prepare this channel. does NOT guarantee to created mChannels[] entry!
+		void OpenNewChannel_(const std::string & channel);  ///< internal function, will throw in case of problems
 		std::string GetLogBaseDir() const;
 
 		std::map< std::thread::id , int > mThread2Number; // change long thread IDs into a short nice number to show
@@ -360,7 +367,10 @@ eSubjectType String2SubjectType(const string & type);
 class cFilesystemUtils { // if we do not want to use boost in given project (or we could optionally write boost here later)
 	public:
 		static bool CreateDirTree(const std::string & dir, bool only_below=false);
-		static char GetDirSeparator(); // eg '/' or '\'
+		static char GetDirSeparatorSys(); /// < eg '/' or '\'
+		static char GetDirSeparatorInter(); /// < internal is '/'
+		static string FileInternalToSystem(const std::string &name); ///< converts from internal file name string to system file name string
+		static string FileSystemToInternal(const std::string &name); ///< converts from system file name string to internal file name string
 };
 
 

src/common/util.cpp

@@ -326,7 +326,7 @@ std::string get_nix_version_display_string()
     std::string config_folder;
 
 #ifdef WIN32
-    config_folder = get_special_folder_path(CSIDL_APPDATA, true) + "/" + CRYPTONOTE_NAME;
+    config_folder = get_special_folder_path(CSIDL_APPDATA, true) + "\\" + CRYPTONOTE_NAME;
 #else
     std::string pathRet;
     char* pszHome = getenv("HOME");

src/daemon/daemon.cpp

@@ -125,8 +125,9 @@ int main(int argc, char* argv[])
   LOG_PRINT_L0("Starting...");
 
   nOT::nUtils::cFilesystemUtils::CreateDirTree("log/dr-monero/net/");
-	// _warn_c("log/test","Starting program"); // TODO
-
+	_warn_c("test","Starting program (a test message)");
+	_warn_c("main/program","Starting program");
+
   TRY_ENTRY();  
 
   boost::filesystem::path default_data_path {tools::get_default_data_dir()};
@@ -319,6 +320,7 @@ int main(int argc, char* argv[])
   ccore.set_cryptonote_protocol(NULL);
   cprotocol.set_p2p_endpoint(NULL);
 
+  epee::net_utils::data_logger::get_instance().kill_instance();
   LOG_PRINT("Node stopped.", LOG_LEVEL_0);
   return 0;
 

src/p2p/connection_basic.cpp

@@ -272,6 +272,7 @@ void connection_basic::logger_handle_net_write(size_t size) {
 }
 
 double connection_basic::get_sleep_time(size_t cb) {
+	CRITICAL_REGION_LOCAL(epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_out);
     auto t = network_throttle_manager::get_global_throttle_out().get_sleep_time(cb);
     return t;
 }