Lots of changes to improve debugging mutex-problems.

It's better to core-dump when mutex calls fails, since it should never happen.
Note: Make sure to turn on coredump-support before starting binary:
   ulimit -c unlimited ; owserver --foreground
--disable-mutexdebug should perhaps be used on smaller embedded systems.
rwlocks are imporoved to used pthread-functions by default.

.gitignore

@@ -15,6 +15,7 @@
 *.so
 *.so.*
 *.dylib
+*.lint
 
 # Executables
 *.exe

Makefile.am

@@ -1,8 +1,10 @@
+ACLOCAL_AMFLAGS = -I src/scripts/m4
+
 SUBDIRS = src module
 RPMMACROS = ${HOME}/.rpmmacros
 RPMDIR = ${HOME}/.rpm
 
-EXTRA_DIST = bootstrap acx_pthread.m4
+EXTRA_DIST = bootstrap
 
 clean-generic:
 

bootstrap

@@ -1,4 +1,4 @@
 #! /bin/sh
 
 rm -rf *cache
-aclocal && autoheader && autoconf && automake && echo "Run './configure' and 'make' to build owfs"
+libtoolize -f -c -i && aclocal -I src/scripts/m4 && autoheader && autoconf && automake -a -c --foreign && echo "Run './configure' and 'make' to build owfs"

configure.ac

@@ -46,8 +46,7 @@ AC_MSG_RESULT(Configuring ${PACKAGE}-${VERSION})
 
 AC_CONFIG_AUX_DIR(src/scripts/install)dnl
 
-#added based on libtoolize warning output
-AC_CONFIG_MACRO_DIR([m4])dnl
+AC_CONFIG_MACRO_DIR([src/scripts/m4])
 
 AC_CANONICAL_TARGET
 AM_INIT_AUTOMAKE(${PACKAGE}, ${VERSION})dnl
@@ -80,11 +79,6 @@ AC_PROG_INSTALL
 AC_LIBTOOL_WIN32_DLL
 AC_PROG_LN_S
 AC_PROG_MAKE_SET
-
-# Modern libtool prefers LT_INIT over AC_PROG_RABLIB
-#AC_PROG_RANLIB
-LT_INIT
-
 AC_PROG_AWK
 AC_LIBTOOL_DLOPEN
 AM_PROG_LIBTOOL
@@ -170,7 +164,7 @@ fi
 OWFSROOT="`pwd`"
 AC_SUBST(OWFSROOT)
 
-m4_include(acx_pthread.m4)
+m4_include([src/scripts/m4/acx_pthread.m4])
 
 # Checks for header files.
 AC_HEADER_DIRENT
@@ -195,6 +189,23 @@ AC_ARG_ENABLE(debug,
 AC_SUBST(ENABLE_DEBUG)
 AM_CONDITIONAL(ENABLE_DEBUG, test "${ENABLE_DEBUG}" = "true")
 
+# Test if mutex-debugging is enabled. Should not be enabled for smaller embedded systems
+ENABLE_MUTEX_DEBUG="true"
+AC_MSG_CHECKING([if mutexdebug is enabled])
+AC_ARG_ENABLE(mutexdebug,
+[  --enable-mutexdebug     Enable mutexdebug-output (default true)],
+[
+	AC_MSG_RESULT([$enableval])
+	if ! test "$enableval" = "yes" ; then
+		ENABLE_MUTEX_DEBUG="false"
+	fi
+],
+[
+	AC_MSG_RESULT([yes (default)])
+])
+AC_SUBST(ENABLE_MUTEX_DEBUG)
+AM_CONDITIONAL(ENABLE_MUTEX_DEBUG, test "${ENABLE_MUTEX_DEBUG}" = "true")
+
 # Test if OWSHELL should be supported
 ENABLE_OWSHELL="true"
 AC_MSG_CHECKING([if OWSHELL support is enabled])
@@ -1514,6 +1525,13 @@ else
 fi
 AC_SUBST(OW_DEBUG)
 
+if test "${ENABLE_MUTEX_DEBUG}" = "true" ; then
+	OW_MUTEX_DEBUG=1
+else
+	OW_MUTEX_DEBUG=0
+fi
+AC_SUBST(OW_MUTEX_DEBUG)
+
 if test "${ENABLE_OWMALLOC}" = "true" ; then
 	OW_ALLOC_DEBUG=1
 else
@@ -1704,6 +1722,11 @@ if test "${ENABLE_DEBUG}" = "true"; then
 else
 	AC_MSG_RESULT([             Debug-output is DISABLED])
 fi
+if test "${ENABLE_MUTEX_DEBUG}" = "true"; then
+	AC_MSG_RESULT([               Mutexdebug is enabled])
+else
+	AC_MSG_RESULT([               Mutexdebug is DISABLED])
+fi
 if test "${ENABLE_PROFILING}" = "true"; then
 	AC_MSG_RESULT([                Profiling is enabled])
 else

module/owlib/src/c/Makefile.am

@@ -248,3 +248,18 @@ AM_CFLAGS = -I../include \
 	${LIBUSB_CFLAGS} \
 	${PIC_FLAGS}
 
+
+if HAVE_CYGWIN
+NOWINE=1
+else
+DOS_OWFSROOT:=$(OWFSROOT)
+LINT_DIR:=$(DOS_OWFSROOT)/src/tools/lint
+LINT_CC:=wine $(LINT_DIR)/lint-nt.exe
+endif
+
+SRC:=$(OWLIB_SOURCE)
+include $(abs_top_srcdir)/src/tools/lint/lint.mk
+
+clean-generic:
+	@RM@ -f *~ .*~ *.lint *.lint.txt .#* *.bak
+

module/owlib/src/c/error.c

@@ -29,7 +29,6 @@
 
 /* module/ownet/c/src/c/error.c & module/owlib/src/c/error.c are identical */
 
-const char sem_init_failed[] = "semaphore_init failed rc=%d [%s]\n";
 const char mutex_init_failed[] = "mutex_init failed rc=%d [%s]\n";
 const char mutex_destroy_failed[] = "mutex_destroy failed rc=%d [%s]\n";
 const char mutex_lock_failed[] = "mutex_lock failed rc=%d [%s]\n";
@@ -42,9 +41,16 @@ const char rwlock_read_lock_failed[] = "rwlock_read_lock failed rc=%d [%s]\n";
 const char rwlock_read_unlock_failed[] = "rwlock_read_unlock failed rc=%d [%s]\n";
 const char cond_timedwait_failed[] = "cond_timedwait failed rc=%d [%s]\n";
 const char cond_signal_failed[] = "cond_signal failed rc=%d [%s]\n";
+const char cond_broadcast_failed[] = "cond_broadcast failed rc=%d [%s]\n";
 const char cond_wait_failed[] = "cond_wait failed rc=%d [%s]\n";
 const char cond_init_failed[] = "cond_init failed rc=%d [%s]\n";
 const char cond_destroy_failed[] = "cond_destroy failed rc=%d [%s]\n";
+const char sem_init_failed[] = "sem_init failed rc=%d [%s]\n";
+const char sem_post_failed[] = "sem_post failed rc=%d [%s]\n";
+const char sem_wait_failed[] = "sem_wait failed rc=%d [%s]\n";
+const char sem_trywait_failed[] = "sem_trywait failed rc=%d [%s]\n";
+const char sem_timedwait_failed[] = "sem_timedwait failed rc=%d [%s]\n";
+const char sem_destroy_failed[] = "sem_destroy failed rc=%d [%s]\n";
 
 static void err_format(char * format, int errno_save, const char * level_string, const char * file, int line, const char * func, const char * fmt);
 static void hex_print( const char * buf, int length ) ;
@@ -61,6 +67,27 @@ int log_available = 0;
  * Caller specifies "errnoflag" and "level" */
 #define MAXLINE     1023
 
+void print_timestamp_(const char * file, int line, const char * func, const char *fmt, ...)
+{
+	struct timeval tv;
+	char buf[MAXLINE + 3];
+	char format[MAXLINE + 3];
+	va_list va;
+
+	gettimeofday(&tv, NULL);
+	va_start(va, fmt);
+	snprintf(format, MAXLINE, "%s:%s(%d) %s", file,func,line,fmt);  /* safe */
+
+#ifdef HAVE_VSNPRINTF
+	vsnprintf(buf, MAXLINE, format, va);    /* safe */
+#else
+	vsprintf(buf, format, va);      /* not safe */
+#endif
+
+	fprintf(stderr, "%ld DEFAULT: %s %ld.%06ld\n", time(NULL), buf, tv.tv_sec, tv.tv_usec);
+	fflush(stderr);
+}
+
 void err_msg(enum e_err_type errnoflag, enum e_err_level level, const char * file, int line, const char * func, const char *fmt, ...)
 {
 	int errno_save = (errnoflag==e_err_type_error)?errno:0;		/* value caller might want printed */
@@ -253,7 +280,8 @@ void fatal_error(const char * file, int line, const char * func, const char *fmt
 	}
 #endif /* OWNETC_OW_DEBUG */
 	va_end(ap);
-	exit(EXIT_FAILURE) ;
+	debug_crash(); // Core-dump to make it possible to trace down the problem!
+	//exit(EXIT_FAILURE) ;
 }
 
 static void err_format(char * format, int errno_save, const char * level_string, const char * file, int line, const char * func, const char * fmt)

module/owlib/src/c/globals.c

@@ -49,7 +49,7 @@ struct global Globals = {
 	.error_level = e_err_default,
 	.error_level_restore = e_err_default,
 	.error_print = e_err_print_mixed,
-	.fatal_debug = 0,
+	.fatal_debug = 1,
 	.fatal_debug_file = NULL,
 
 	.concurrent_connections = 10,

module/owlib/src/c/ow_locks.c

@@ -35,8 +35,8 @@ void LockSetup(void)
 	/* global mutex attribute */
 	_MUTEX_ATTR_INIT(Mutex.mattr);
 
-  #ifdef __UCLIBC__
-    #if ((__UCLIBC_MAJOR__ << 16)+(__UCLIBC_MINOR__ << 8)+(__UCLIBC_SUBLEVEL__) < 0x00091D)
+#ifdef __UCLIBC__
+#if ((__UCLIBC_MAJOR__ << 16)+(__UCLIBC_MINOR__ << 8)+(__UCLIBC_SUBLEVEL__) < 0x00091D)
 	/* If uClibc < 0.9.29, then re-initialize internal pthread-structs
 	 * pthread and mutexes doesn't work after daemon() is called and
 	 *   the main-process is gone.
@@ -47,22 +47,31 @@ void LockSetup(void)
 	__pthread_initial_thread_bos = NULL;
 	__pthread_initialize();
 
+	/* global mutex attribute */
 	_MUTEX_ATTR_SET(Mutex.mattr, PTHREAD_MUTEX_ADAPTIVE_NP);
-    #else /* UCLIBC_VERSION */
+#else /* UCLIBC_VERSION */
 	_MUTEX_ATTR_SET(Mutex.mattr, PTHREAD_MUTEX_DEFAULT);
-    #endif							/* UCLIBC_VERSION */
+#endif							/* UCLIBC_VERSION */
+#define MUTEX_ATTR_INIT_DONE
 	_MUTEX_INIT(Mutex.uclibc_mutex);
-  #else /* __UCLIBC__ */
-	_MUTEX_ATTR_SET(Mutex.mattr, PTHREAD_MUTEX_DEFAULT);   
-  #endif							/* __UCLIBC__ */
+#endif							/* __UCLIBC__ */
 
+#ifndef MUTEX_ATTR_INIT_DONE
+#define MUTEX_ATTR_INIT_DONE
+#ifdef EXTENDED_MUTEX_DEBUG
+	_MUTEX_ATTR_SET(Mutex.mattr, PTHREAD_MUTEX_ERRORCHECK);
+#else
+	_MUTEX_ATTR_SET(Mutex.mattr, PTHREAD_MUTEX_DEFAULT);
+#endif
+#endif
+
 	_MUTEX_INIT(Mutex.stat_mutex);
 	_MUTEX_INIT(Mutex.controlflags_mutex);
 	_MUTEX_INIT(Mutex.fstat_mutex);
 	_MUTEX_INIT(Mutex.dir_mutex);
-  #if OW_USB
+#if OW_USB
 	_MUTEX_INIT(Mutex.libusb_mutex);
-  #endif							/* OW_USB */
+#endif							/* OW_USB */
 	_MUTEX_INIT(Mutex.typedir_mutex);
 	_MUTEX_INIT(Mutex.externaldir_mutex);
 	_MUTEX_INIT(Mutex.namefind_mutex);

module/owlib/src/c/ow_opt.c

@@ -106,6 +106,8 @@ const struct option owopts_long[] = {
 	{"foreground", no_argument, NO_LINKED_VAR, e_want_foreground},
 	{"fatal_debug", no_argument, &Globals.fatal_debug, 1},
 	{"fatal-debug", no_argument, &Globals.fatal_debug, 1},
+	{"nofatal_debug", no_argument, &Globals.fatal_debug, 0},
+	{"nofatal-debug", no_argument, &Globals.fatal_debug, 0},
 	{"fatal_debug_file", required_argument, NO_LINKED_VAR, e_fatal_debug_file},
 	{"fatal-debug-file", required_argument, NO_LINKED_VAR, e_fatal_debug_file},
 	{"error_print", required_argument, NO_LINKED_VAR, e_error_print},