Support fuzz testing (#4)

Quickstart (requires Clang 6 or later):

$ export LIB_FUZZING_ENGINE=/path/to/fuzzing/library.a
$ ./configure --enable-fuzz-testing
$ make
$ cd src/fuzz
$ make fuzz_mdb
$ ./fuzz_mdb

Also add a new `mdb_open_buffer function` to facilitate in-memory
fuzz-testing. This requires fmemopen, which may not be present on all
systems. The internal API has been reworked to use file streams instead
of file descriptors. This allows reading from memory and reading from
files using a consistent API.

README.md

@@ -23,6 +23,7 @@ Intended focus areas of this fork:
 
 - [ ] Security / stability / fuzz testing
 - [x] Thread safety\*
+- [x] In-memory database API
 - [x] Removing GLib dependency
 - [x] Improved ODBC compliance
 - [x] Continuous integration with Travis and AppVeyor

appveyor.yml

@@ -39,9 +39,19 @@ test_script:
       if ($env:TOOLCHAIN -eq "msys2")
       {
         $env:MSYSTEM="MINGW64"
+        C:\msys64\usr\bin\bash -l -c "cd /c/projects/mdbtools && ./src/util/mdb-ver ./test/data/ASampleDatabase.accdb"
+        C:\msys64\usr\bin\bash -l -c "cd /c/projects/mdbtools && ./src/util/mdb-ver ./test/data/nwind.mdb"
+        C:\msys64\usr\bin\bash -l -c "cd /c/projects/mdbtools && ./src/util/mdb-tables ./test/data/ASampleDatabase.accdb"
+        C:\msys64\usr\bin\bash -l -c "cd /c/projects/mdbtools && ./src/util/mdb-tables ./test/data/nwind.mdb"
+        C:\msys64\usr\bin\bash -l -c "cd /c/projects/mdbtools && ./src/util/mdb-count ./test/data/ASampleDatabase.accdb 'Asset Items'"
         C:\msys64\usr\bin\bash -l -c "cd /c/projects/mdbtools && ./src/util/mdb-count ./test/data/nwind.mdb Customers"
       }
       else
       {
+        C:\cygwin64\bin\sh -lc "cd /cygdrive/c/projects/mdbtools && ./src/util/mdb-ver ./test/data/ASampleDatabase.accdb"
+        C:\cygwin64\bin\sh -lc "cd /cygdrive/c/projects/mdbtools && ./src/util/mdb-ver ./test/data/nwind.mdb"
+        C:\cygwin64\bin\sh -lc "cd /cygdrive/c/projects/mdbtools && ./src/util/mdb-tables ./test/data/ASampleDatabase.accdb"
+        C:\cygwin64\bin\sh -lc "cd /cygdrive/c/projects/mdbtools && ./src/util/mdb-tables ./test/data/nwind.mdb"
+        C:\cygwin64\bin\sh -lc "cd /cygdrive/c/projects/mdbtools && ./src/util/mdb-count ./test/data/ASampleDatabase.accdb 'Asset Items'"
         C:\cygwin64\bin\sh -lc "cd /cygdrive/c/projects/mdbtools && ./src/util/mdb-count ./test/data/nwind.mdb Customers"
       }

configure.ac

@@ -9,6 +9,7 @@ AM_MAINTAINER_MODE([enable])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
 AC_PROG_CC(gcc)
+AC_PROG_CXX
 dnl Checks for programs.
 AC_PROG_MAKE_SET
 m4_pattern_allow([AM_PROG_AR], [AM_PROG_AR])
@@ -25,6 +26,11 @@ AC_CHECK_DECLS([program_invocation_short_name], [], [], [[
                 #define _GNU_SOURCE
                 #include <errno.h>]])
 
+dnl Checks for library functions.
+VL_LIB_READLINE
+AC_CHECK_FUNC(strptime,[ AC_DEFINE(HAVE_STRPTIME, 1, [strptime check]) AM_CONDITIONAL(HAVE_STRPTIME, true) ],[ AM_CONDITIONAL(HAVE_STRPTIME, false) ])
+AC_CHECK_FUNC(fmemopen,[ AC_DEFINE(HAVE_FMEMOPEN, 1, [fmemopen check]) ])
+
 dnl Checks for typedefs, structures, and compiler characteristics.
 AC_C_CONST
 AC_TYPE_SIZE_T
@@ -76,6 +82,27 @@ AS_CASE([$host],
         [*mingw*|*cygwin*], [LDFLAGS="$LDFLAGS -no-undefined"], [])
 AS_CASE([$host],
         [*mingw*], [LDFLAGS="$LDFLAGS -lWs2_32"], [])
+
+dnl Fuzz testing
+
+AC_ARG_ENABLE([fuzz-testing], AS_HELP_STRING([--enable-fuzz-testing], ["Enable fuzz testing (requires Clang 6 or later)"]), [
+   AC_MSG_CHECKING([whether $CC accepts -fsanitize=fuzzer])
+   tmp_saved_flags=$[]_AC_LANG_PREFIX[]FLAGS
+   _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS -fsanitize=fuzzer"
+   AC_LINK_IFELSE([AC_LANG_PROGRAM()],
+       [
+        AC_MSG_RESULT(yes)
+        OPTDIRS="$OPTDIRS fuzz"
+        fuzzer=yes],
+        AC_MSG_RESULT(no)
+       AC_MSG_FAILURE([-fsanitize=fuzzer not supported (Required with --enable-fuzz-testing)]))
+   _AC_LANG_PREFIX[]FLAGS=$tmp_saved_flags
+], [fuzzer=no])
+AM_CONDITIONAL([FUZZER_ENABLED], test "x$fuzzer" = "xyes")
+
+AC_ARG_VAR([LIB_FUZZING_ENGINE], [Location of prebuilt fuzzing engine library])
+AC_SUBST([LIB_FUZZING_ENGINE])
+
 dnl Enable -Wl,--as-needed by default to prevent overlinking
 
 AC_ARG_ENABLE([as-needed],
@@ -269,10 +296,6 @@ AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" = yes)
 ##################################################
 AM_CONDITIONAL(ENABLE_DOCBOOK, test -n "$DOCBOOK_DSL")
 
-dnl Checks for library functions.
-VL_LIB_READLINE
-AC_CHECK_FUNC(strptime,[ AC_DEFINE(HAVE_STRPTIME, 1, [strptime check]) AM_CONDITIONAL(HAVE_STRPTIME, true) ],[ AM_CONDITIONAL(HAVE_STRPTIME, false) ])
-
 localedir=${datadir}/locale
 AC_SUBST(localedir)
 
@@ -287,6 +310,7 @@ include/Makefile
 src/libmdb/Makefile
 src/sql/Makefile
 src/odbc/Makefile
+src/fuzz/Makefile
 doc/Makefile
 src/gmdb2/Makefile
 src/gmdb2/gladefiles/Makefile

include/mdbtools.h

@@ -225,9 +225,8 @@ typedef struct {
 } MdbStatistics;
 
 typedef struct {
-	int           fd;
+	FILE        *stream;
 	gboolean      writable;
-	char          *filename;
 	guint32		jet_version;
 	guint32		db_key;
 	char		db_passwd[14];
@@ -463,6 +462,7 @@ long   mdb_pg_get_int32(MdbHandle *mdb, int offset);
 float  mdb_pg_get_single(MdbHandle *mdb, int offset);
 double mdb_pg_get_double(MdbHandle *mdb, int offset);
 MdbHandle *mdb_open(const char *filename, MdbFileFlags flags);
+MdbHandle *mdb_open_buffer(void *buffer, size_t len, MdbFileFlags flags);
 void mdb_close(MdbHandle *mdb);
 MdbHandle *mdb_clone_handle(MdbHandle *mdb);
 void mdb_swap_pgbuf(MdbHandle *mdb);

src/fuzz/Makefile.am

@@ -0,0 +1,9 @@
+EXTRA_PROGRAMS = fuzz_mdb
+
+# Force C++ linking for fuzz target
+nodist_EXTRA_fuzz_mdb_SOURCES = dummy.cxx
+fuzz_mdb_SOURCES = fuzz_mdb.c
+
+AM_CFLAGS = -I$(top_srcdir)/include $(GLIB_CFLAGS) -g
+AM_LDFLAGS = -static
+LDADD = ../libmdb/libmdb.la @LIB_FUZZING_ENGINE@

src/fuzz/fuzz_mdb.c

@@ -0,0 +1,20 @@
+#include "mdbtools.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+    MdbHandle *mdb = mdb_open_buffer((void *)Data, Size, MDB_NOFLAGS);
+    if (mdb) {
+        mdb_read_catalog(mdb, MDB_TABLE);
+        for (int j=0; j<mdb->num_catalog; j++) {
+            MdbCatalogEntry *entry = g_ptr_array_index (mdb->catalog, j);
+            MdbTableDef *table = mdb_read_table(entry);
+            if (table) {
+                mdb_read_columns(table);
+                mdb_rewind_table(table);
+                while (mdb_fetch_row(table));
+                mdb_free_tabledef(table);
+            }
+        }
+        mdb_close(mdb);
+    }
+    return 0;
+}

src/libmdb/Makefile.am

@@ -5,4 +5,8 @@ libmdb_la_SOURCES += fakeglib.c
 endif
 libmdb_la_LDFLAGS = -version-info 2:1:0 -export-symbols-regex '^(mdb_|_mdb_put_int16$$|_mdb_put_int32$$)'
 AM_CFLAGS	=	-I$(top_srcdir)/include $(GLIB_CFLAGS) -Wsign-compare
+if FUZZER_ENABLED
+AM_CFLAGS += -fsanitize=fuzzer-no-link -fsanitize=address
+libmdb_la_LDFLAGS += -fsanitize=fuzzer -fsanitize=address
+endif
 LIBS = $(GLIB_LIBS) @LIBS@ @LIBICONV@

src/libmdb/catalog.c

@@ -93,6 +93,7 @@ GPtrArray *mdb_read_catalog (MdbHandle *mdb, int objtype)
 
 	table = mdb_read_table(&msysobj);
     if (!table) {
+        fprintf(stderr, "Unable to read table %s\n", msysobj.object_name);
         mdb_free_catalog(mdb);
         goto cleanup;
     }
@@ -103,10 +104,13 @@ GPtrArray *mdb_read_catalog (MdbHandle *mdb, int objtype)
         mdb_bind_column_by_name(table, "Name", obj_name, NULL) == -1 ||
         mdb_bind_column_by_name(table, "Type", obj_type, NULL) == -1 ||
         mdb_bind_column_by_name(table, "Flags", obj_flags, NULL) == -1) {
+        fprintf(stderr, "Unable to bind columns from table %s (%d columns found)\n",
+                msysobj.object_name, table->num_cols);
         mdb_free_catalog(mdb);
         goto cleanup;
     }
     if ((i = mdb_bind_column_by_name(table, "LvProp", obj_props, &kkd_size_ole)) == -1) {
+        fprintf(stderr, "Unable to bind column %s from table %s\n", "LvProp", msysobj.object_name);
         mdb_free_catalog(mdb);
         goto cleanup;
     }