diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..f8c0901 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,32 @@ +======= +CONTACT +======= +For questions regarding SILT, please contact the FAWN/SILT developers at +fawn-dev@mailman.srv.cs.cmu.edu. + + +======= +AUTHORS +======= + +SILT authors: + +Hyeontaek Lim +Bin Fan + +FAWN-KV authors: + +David Andersen +Alex Crichton +Jack Ferris +Jason Franklin +Alex Gartrell +Dongsu Han +Michael Kaminsky +Wyatt Lloyd +Iulian Moraru +Amar Phanishayee +Lawrence Tan +Vijay Vasudevan +Reinhard Munz + diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..156edd0 --- /dev/null +++ b/COPYING @@ -0,0 +1,16 @@ +SILT: A Memory-Efficient, High-Performance Key-Value Store + +Copyright 2011 Carnegie Mellon University + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..000378b --- /dev/null +++ b/INSTALL @@ -0,0 +1,6 @@ +0) # requirements: Intel TBB, Google Test Library, PThread, Boost, OpenSSL, Xerces, GSL, NSort, BDB +1) autoreconf -is +2) ./configure +3) make +4) cd test/fawnds/ +5) ./test diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..54a2928 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,8 @@ +ACLOCAL_AMFLAGS=-I m4 + +SUBDIRS = utils fawnds test +CLEANFILES = core *.core *~ +DISTCLEANFILES = autom4te*.cache config.status config.log +MAINTAINERCLEANFILES = aclocal.m4 install-sh mkinstalldirs \ + missing configure config.guess config.sub config.h.in \ + ltconfig ltmain.sh Makefile.in stamp-h.in depcomp m4/*.m4 diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..e69de29 diff --git a/README b/README new file mode 100644 index 0000000..e69de29 diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..ed6157c --- /dev/null +++ b/configure.ac @@ -0,0 +1,81 @@ +AC_PREREQ(2.61) +AC_INIT([silt], [0.1], [fawn-dev@mailman.srv.cs.cmu.edu]) +AM_INIT_AUTOMAKE +LT_PREREQ([2.2]) +LT_INIT([dlopen]) + +AC_CONFIG_HEADER([config.h]) + +CFLAGS="-g -O3" # -fno-omit-frame-pointer +CPPFLAGS="-g -O3" # -fno-omit-frame-pointer +CXXFLAGS="-g -O3" # -fno-omit-frame-pointer + +# Checks for programs. +AC_PROG_CXX +AC_PROG_CC +AC_PROG_MAKE_SET + +AC_PROG_LIBTOOL +AC_CONFIG_MACRO_DIR([m4]) + + +# Checks for libraries. +# FIXME: Replace `main' with a function in `-lgtest': +AC_CHECK_LIB([gtest], [main], [LIBS="-lgtest $LIBS"; no_libgtest=false], + [echo "gtest unittesting framework not found."; no_libgtest=true]) +AC_LANG_CPLUSPLUS +AC_CHECK_LIB([tbb], [main], [], [echo "libtbb not found. Please install the Intel TBB before proceeding."; exit -1]) +AC_CHECK_LIB([gtest_main], [main], [], [echo "gtest_main not found. Please install google test library before proceeding"; exit -1]) +AC_CHECK_LIB([pthread], [pthread_mutex_init], [], [echo "pthreads not found. Please install pthread library before proceeding"; exit -1]) +AC_CHECK_LIB([boost_thread-mt], [main], [], [echo "boost library not found. Please install boost library before proceeding"; exit -1]) +AC_CHECK_LIB([crypto], [main], [], [echo "openssl crypto library not found. Please install openssl library before proceeding"; exit -1]) +AC_CHECK_LIB([xerces-c], [main], [], [echo "Xerces XML parsing library not found. Please install xerces library before proceeding"; exit -1]) +AC_CHECK_LIB([m], [cos], [], [echo "m library not found. Please install m library before proceeding"; exit -1]) +AC_CHECK_LIB([gslcblas], [cblas_dgemm], [], [echo "gsl library not found. Please install gsl library before proceeding"; exit -1]) +AC_CHECK_LIB([gsl], [gsl_blas_dgemm], [], [echo "gsl library not found. Please install gsl library before proceeding"; exit -1]) +AC_CHECK_LIB([nsort], [nsort_define], [], [echo "nsort library not found. An inferior sorting will be used."]) +AC_CHECK_LIB([db], [db_create], [], [echo "Berkeley Database library not found. Support for BDB will be excluded."]) + + +# Checks for header files. +AC_HEADER_STDC +AC_CHECK_HEADERS([arpa/inet.h fcntl.h netdb.h netinet/in.h stdlib.h string.h sys/ioctl.h sys/param.h sys/socket.h sys/time.h unistd.h]) + +# Checks for typedefs, structures, and compiler characteristics. +AC_HEADER_STDBOOL +AC_C_CONST +AC_C_INLINE +AC_TYPE_INT32_T +AC_TYPE_INT64_T +AC_TYPE_OFF_T +AC_TYPE_SIZE_T +AC_TYPE_SSIZE_T +AC_HEADER_TIME +AC_TYPE_UINT16_T +AC_TYPE_UINT32_T +AC_TYPE_UINT64_T +AC_TYPE_UINT8_T +AC_C_VOLATILE + + +# Checks for library functions. +AC_PROG_GCC_TRADITIONAL +AC_FUNC_MALLOC +AC_FUNC_MEMCMP +AC_FUNC_MMAP +AC_FUNC_REALLOC +AC_FUNC_SELECT_ARGTYPES +AC_TYPE_SIGNAL +AC_CHECK_FUNCS([bzero ftruncate gethostbyname getpagesize gettimeofday inet_ntoa memchr memmove memset munmap select socket strdup strerror strtol strtoul strtoull]) + +#CFLAGS="$CFLAGS -Wall -Wextra" +#CXXFLAGS="$CXXFLAGS -Wall -Wextra" + +AC_CONFIG_FILES([ Makefile + utils/Makefile + fawnds/Makefile + test/Makefile + test/fawnds/Makefile + ]) + +AC_OUTPUT diff --git a/fawnds/DOMTreeErrorReporter.cpp b/fawnds/DOMTreeErrorReporter.cpp new file mode 100644 index 0000000..d011705 --- /dev/null +++ b/fawnds/DOMTreeErrorReporter.cpp @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * $Id: DOMTreeErrorReporter.cpp 471735 2006-11-06 13:53:58Z amassari $ + */ + +// --------------------------------------------------------------------------- +// Includes +// --------------------------------------------------------------------------- +#include +#include "DOMTreeErrorReporter.hpp" +#if defined(XERCES_NEW_IOSTREAMS) +#include +#else +#include +#endif +#include +#include + + +void DOMTreeErrorReporter::warning(const SAXParseException& toCatch) +{ + XERCES_STD_QUALIFIER cerr << "Warning at file \"" << StrX(toCatch.getSystemId()) + << "\", line " << toCatch.getLineNumber() + << ", column " << toCatch.getColumnNumber() + << "\n Message: " << StrX(toCatch.getMessage()) << XERCES_STD_QUALIFIER endl; +} + +void DOMTreeErrorReporter::error(const SAXParseException& toCatch) +{ + fSawErrors = true; + XERCES_STD_QUALIFIER cerr << "Error at file \"" << StrX(toCatch.getSystemId()) + << "\", line " << toCatch.getLineNumber() + << ", column " << toCatch.getColumnNumber() + << "\n Message: " << StrX(toCatch.getMessage()) << XERCES_STD_QUALIFIER endl; +} + +void DOMTreeErrorReporter::fatalError(const SAXParseException& toCatch) +{ + fSawErrors = true; + XERCES_STD_QUALIFIER cerr << "Fatal Error at file \"" << StrX(toCatch.getSystemId()) + << "\", line " << toCatch.getLineNumber() + << ", column " << toCatch.getColumnNumber() + << "\n Message: " << StrX(toCatch.getMessage()) << XERCES_STD_QUALIFIER endl; +} + +void DOMTreeErrorReporter::resetErrors() +{ + fSawErrors = false; +} + + diff --git a/fawnds/DOMTreeErrorReporter.hpp b/fawnds/DOMTreeErrorReporter.hpp new file mode 100644 index 0000000..145de89 --- /dev/null +++ b/fawnds/DOMTreeErrorReporter.hpp @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * $Id: DOMTreeErrorReporter.hpp 471735 2006-11-06 13:53:58Z amassari $ + */ + +#include +#include +#if defined(XERCES_NEW_IOSTREAMS) +#include +#else +#include +#endif + + +XERCES_CPP_NAMESPACE_USE + + +class DOMTreeErrorReporter : public ErrorHandler +{ +public: + // ----------------------------------------------------------------------- + // Constructors and Destructor + // ----------------------------------------------------------------------- + DOMTreeErrorReporter() : + fSawErrors(false) + { + } + + ~DOMTreeErrorReporter() + { + } + + + // ----------------------------------------------------------------------- + // Implementation of the error handler interface + // ----------------------------------------------------------------------- + void warning(const SAXParseException& toCatch); + void error(const SAXParseException& toCatch); + void fatalError(const SAXParseException& toCatch); + void resetErrors(); + + // ----------------------------------------------------------------------- + // Getter methods + // ----------------------------------------------------------------------- + bool getSawErrors() const; + + // ----------------------------------------------------------------------- + // Private data members + // + // fSawErrors + // This is set if we get any errors, and is queryable via a getter + // method. Its used by the main code to suppress output if there are + // errors. + // ----------------------------------------------------------------------- + bool fSawErrors; +}; + +inline bool DOMTreeErrorReporter::getSawErrors() const +{ + return fSawErrors; +} + +// --------------------------------------------------------------------------- +// This is a simple class that lets us do easy (though not terribly efficient) +// trancoding of XMLCh data to local code page for display. +// --------------------------------------------------------------------------- +class StrX +{ +public : + // ----------------------------------------------------------------------- + // Constructors and Destructor + // ----------------------------------------------------------------------- + StrX(const XMLCh* const toTranscode) + { + // Call the private transcoding method + fLocalForm = XMLString::transcode(toTranscode); + } + + ~StrX() + { + XMLString::release(&fLocalForm); + } + + + // ----------------------------------------------------------------------- + // Getter methods + // ----------------------------------------------------------------------- + const char* localForm() const + { + return fLocalForm; + } + +private : + // ----------------------------------------------------------------------- + // Private data members + // + // fLocalForm + // This is the local code page form of the string. + // ----------------------------------------------------------------------- + char* fLocalForm; +}; + +inline XERCES_STD_QUALIFIER ostream& operator<<(XERCES_STD_QUALIFIER ostream& target, const StrX& toDump) +{ + target << toDump.localForm(); + return target; +} + diff --git a/fawnds/DOMWriteErrorHandler.cpp b/fawnds/DOMWriteErrorHandler.cpp new file mode 100644 index 0000000..5728bb5 --- /dev/null +++ b/fawnds/DOMWriteErrorHandler.cpp @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * $Id: DOMPrintErrorHandler.cpp 471735 2006-11-06 13:53:58Z amassari $ + */ + +#include +#include +#if defined(XERCES_NEW_IOSTREAMS) +#include +#else +#include +#endif + +#include "DOMWriteErrorHandler.hpp" + +bool DOMWriteErrorHandler::handleError(const DOMError &domError) +{ + // Display whatever error message passed from the serializer + if (domError.getSeverity() == DOMError::DOM_SEVERITY_WARNING) + XERCES_STD_QUALIFIER cerr << "\nWarning Message: "; + else if (domError.getSeverity() == DOMError::DOM_SEVERITY_ERROR) + XERCES_STD_QUALIFIER cerr << "\nError Message: "; + else + XERCES_STD_QUALIFIER cerr << "\nFatal Message: "; + + char *msg = XMLString::transcode(domError.getMessage()); + XERCES_STD_QUALIFIER cerr<< msg < + +XERCES_CPP_NAMESPACE_USE + +class DOMWriteErrorHandler : public DOMErrorHandler +{ +public: + + DOMWriteErrorHandler(){}; + ~DOMWriteErrorHandler(){}; + + /** @name The error handler interface */ + bool handleError(const DOMError& domError); + void resetErrors(){}; + +private : + /* Unimplemented constructors and operators */ + DOMWriteErrorHandler(const DOMErrorHandler&); + void operator=(const DOMErrorHandler&); + +}; + +#endif // #ifndef DOM_WRITE_ERROR_HANDLER_HPP diff --git a/fawnds/Makefile.am b/fawnds/Makefile.am new file mode 100644 index 0000000..2c3dad7 --- /dev/null +++ b/fawnds/Makefile.am @@ -0,0 +1,77 @@ +noinst_LTLIBRARIES = libfawnds.la +libfawnds_la_CPPFLAGS = -I$(top_srcdir)/utils -D_FILE_OFFSET_BITS=64 \ + -I$(builddir)/gen-cpp -Icindex -g +libfawnds_la_LIBADD = $(top_builddir)/utils/libfawnkvutils.la + + +noinst_HEADERS = DOMTreeErrorReporter.hpp \ + DOMWriteErrorHandler.hpp \ + configuration.h \ + file_io.h \ + task.h \ + value.h \ + rate_limiter.h \ + global_limits.h \ + fawnds_types.h \ + fawnds_iterator.h \ + fawnds.h \ + fawnds_factory.h \ + hash_functions.h \ + hash_table_default.h \ + hash_table_cuckoo.h \ + file_store.h \ + sorter.h \ + fawnds_partition.h \ + fawnds_combi.h \ + fawnds_sf.h \ + fawnds_sf_ordered_trie.h \ + fawnds_proxy.h \ + fawnds_monitor.h \ + bdb.h + +libfawnds_la_SOURCES = DOMTreeErrorReporter.cpp \ + DOMWriteErrorHandler.cpp \ + configuration.cc \ + file_io.cc \ + task.cc \ + rate_limiter.cc \ + global_limits.cc \ + fawnds_iterator.cc \ + fawnds.cc \ + fawnds_factory.cc \ + hash_functions.cc \ + hash_table_default.cc \ + hash_table_cuckoo.cc \ + file_store.cc \ + sorter.cc \ + fawnds_partition.cc \ + fawnds_combi.cc \ + fawnds_sf.cc \ + fawnds_sf_ordered_trie.cc \ + fawnds_proxy.cc \ + fawnds_monitor.cc \ + bdb.cc \ + cindex/bit_access.cpp \ + cindex/bit_vector.cpp \ + cindex/expected_size.cpp \ + cindex/flat_absoff_bucketing.cpp \ + cindex/twolevel_absoff_bucketing.cpp \ + cindex/twolevel_reloff_bucketing.cpp \ + cindex/semi_direct_16_absoff_bucketing.cpp \ + cindex/semi_direct_16_reloff_bucketing.cpp \ + cindex/bucketing_index.cpp + +CLEANFILES = $(BUILT_SOURCES) + + +# kaminsky: following line forces noinst_* libraries to build +# shared. This can help with development because a change to +# this library doesn't require re-building libs and programs +# that link against this library +#libfawnds_la_LDFLAGS = -rpath `pwd` + +noinst_PROGRAMS = fawnds_bench +fawnds_bench_SOURCES = fawnds_bench.cc +fawnds_bench_CPPFLAGS = -I$(top_srcdir)/utils -I$(builddir)/gen-cpp -g +fawnds_bench_LDADD = $(top_builddir)/utils/libfawnkvutils.la libfawnds.la -lxerces-c + diff --git a/fawnds/Makefile.in b/fawnds/Makefile.in new file mode 100644 index 0000000..3c29ea8 --- /dev/null +++ b/fawnds/Makefile.in @@ -0,0 +1,862 @@ +# Makefile.in generated by automake 1.11.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +# 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, +# Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +noinst_PROGRAMS = fawnds_bench$(EXEEXT) +subdir = fawnds +DIST_COMMON = README $(noinst_HEADERS) $(srcdir)/Makefile.am \ + $(srcdir)/Makefile.in +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/libtool.m4 \ + $(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \ + $(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \ + $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libfawnds_la_DEPENDENCIES = $(top_builddir)/utils/libfawnkvutils.la +am_libfawnds_la_OBJECTS = libfawnds_la-DOMTreeErrorReporter.lo \ + libfawnds_la-DOMWriteErrorHandler.lo \ + libfawnds_la-configuration.lo libfawnds_la-file_io.lo \ + libfawnds_la-task.lo libfawnds_la-rate_limiter.lo \ + libfawnds_la-global_limits.lo libfawnds_la-fawnds_iterator.lo \ + libfawnds_la-fawnds.lo libfawnds_la-fawnds_factory.lo \ + libfawnds_la-hash_functions.lo \ + libfawnds_la-hash_table_default.lo \ + libfawnds_la-hash_table_cuckoo.lo libfawnds_la-file_store.lo \ + libfawnds_la-sorter.lo libfawnds_la-fawnds_partition.lo \ + libfawnds_la-fawnds_combi.lo libfawnds_la-fawnds_sf.lo \ + libfawnds_la-fawnds_sf_ordered_trie.lo \ + libfawnds_la-fawnds_proxy.lo libfawnds_la-fawnds_monitor.lo \ + libfawnds_la-bdb.lo libfawnds_la-bit_access.lo \ + libfawnds_la-bit_vector.lo libfawnds_la-expected_size.lo \ + libfawnds_la-flat_absoff_bucketing.lo \ + libfawnds_la-twolevel_absoff_bucketing.lo \ + libfawnds_la-twolevel_reloff_bucketing.lo \ + libfawnds_la-semi_direct_16_absoff_bucketing.lo \ + libfawnds_la-semi_direct_16_reloff_bucketing.lo \ + libfawnds_la-bucketing_index.lo +libfawnds_la_OBJECTS = $(am_libfawnds_la_OBJECTS) +PROGRAMS = $(noinst_PROGRAMS) +am_fawnds_bench_OBJECTS = fawnds_bench-fawnds_bench.$(OBJEXT) +fawnds_bench_OBJECTS = $(am_fawnds_bench_OBJECTS) +fawnds_bench_DEPENDENCIES = $(top_builddir)/utils/libfawnkvutils.la \ + libfawnds.la +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__depfiles_maybe = depfiles +am__mv = mv -f +CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +LTCXXCOMPILE = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +CXXLD = $(CXX) +CXXLINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=link $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +SOURCES = $(libfawnds_la_SOURCES) $(fawnds_bench_SOURCES) +DIST_SOURCES = $(libfawnds_la_SOURCES) $(fawnds_bench_SOURCES) +HEADERS = $(noinst_HEADERS) +ETAGS = etags +CTAGS = ctags +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +MAKEINFO = @MAKEINFO@ +MKDIR_P = @MKDIR_P@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +RANLIB = @RANLIB@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +VERSION = @VERSION@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +lt_ECHO = @lt_ECHO@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +sysconfdir = @sysconfdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libfawnds.la +libfawnds_la_CPPFLAGS = -I$(top_srcdir)/utils -D_FILE_OFFSET_BITS=64 \ + -I$(builddir)/gen-cpp -Icindex -g + +libfawnds_la_LIBADD = $(top_builddir)/utils/libfawnkvutils.la +noinst_HEADERS = DOMTreeErrorReporter.hpp \ + DOMWriteErrorHandler.hpp \ + configuration.h \ + file_io.h \ + task.h \ + value.h \ + rate_limiter.h \ + global_limits.h \ + fawnds_types.h \ + fawnds_iterator.h \ + fawnds.h \ + fawnds_factory.h \ + hash_functions.h \ + hash_table_default.h \ + hash_table_cuckoo.h \ + file_store.h \ + sorter.h \ + fawnds_partition.h \ + fawnds_combi.h \ + fawnds_sf.h \ + fawnds_sf_ordered_trie.h \ + fawnds_proxy.h \ + fawnds_monitor.h \ + bdb.h + +libfawnds_la_SOURCES = DOMTreeErrorReporter.cpp \ + DOMWriteErrorHandler.cpp \ + configuration.cc \ + file_io.cc \ + task.cc \ + rate_limiter.cc \ + global_limits.cc \ + fawnds_iterator.cc \ + fawnds.cc \ + fawnds_factory.cc \ + hash_functions.cc \ + hash_table_default.cc \ + hash_table_cuckoo.cc \ + file_store.cc \ + sorter.cc \ + fawnds_partition.cc \ + fawnds_combi.cc \ + fawnds_sf.cc \ + fawnds_sf_ordered_trie.cc \ + fawnds_proxy.cc \ + fawnds_monitor.cc \ + bdb.cc \ + cindex/bit_access.cpp \ + cindex/bit_vector.cpp \ + cindex/expected_size.cpp \ + cindex/flat_absoff_bucketing.cpp \ + cindex/twolevel_absoff_bucketing.cpp \ + cindex/twolevel_reloff_bucketing.cpp \ + cindex/semi_direct_16_absoff_bucketing.cpp \ + cindex/semi_direct_16_reloff_bucketing.cpp \ + cindex/bucketing_index.cpp + +CLEANFILES = $(BUILT_SOURCES) +fawnds_bench_SOURCES = fawnds_bench.cc +fawnds_bench_CPPFLAGS = -I$(top_srcdir)/utils -I$(builddir)/gen-cpp -g +fawnds_bench_LDADD = $(top_builddir)/utils/libfawnkvutils.la libfawnds.la -lxerces-c +all: all-am + +.SUFFIXES: +.SUFFIXES: .cc .cpp .lo .o .obj +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu fawnds/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --gnu fawnds/Makefile +.PRECIOUS: Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" != "$$p" || dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done +libfawnds.la: $(libfawnds_la_OBJECTS) $(libfawnds_la_DEPENDENCIES) + $(CXXLINK) $(libfawnds_la_OBJECTS) $(libfawnds_la_LIBADD) $(LIBS) + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list +fawnds_bench$(EXEEXT): $(fawnds_bench_OBJECTS) $(fawnds_bench_DEPENDENCIES) + @rm -f fawnds_bench$(EXEEXT) + $(CXXLINK) $(fawnds_bench_OBJECTS) $(fawnds_bench_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fawnds_bench-fawnds_bench.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-DOMTreeErrorReporter.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-DOMWriteErrorHandler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-bdb.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-bit_access.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-bit_vector.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-bucketing_index.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-configuration.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-expected_size.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-fawnds.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-fawnds_combi.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-fawnds_factory.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-fawnds_iterator.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-fawnds_monitor.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-fawnds_partition.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-fawnds_proxy.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-fawnds_sf.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-fawnds_sf_ordered_trie.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-file_io.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-file_store.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-flat_absoff_bucketing.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-global_limits.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-hash_functions.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-hash_table_cuckoo.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-hash_table_default.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-rate_limiter.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-semi_direct_16_absoff_bucketing.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-semi_direct_16_reloff_bucketing.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-sorter.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-task.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-twolevel_absoff_bucketing.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libfawnds_la-twolevel_reloff_bucketing.Plo@am__quote@ + +.cc.o: +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ $< + +.cc.obj: +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.cc.lo: +@am__fastdepCXX_TRUE@ $(LTCXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $< + +libfawnds_la-DOMTreeErrorReporter.lo: DOMTreeErrorReporter.cpp +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-DOMTreeErrorReporter.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-DOMTreeErrorReporter.Tpo -c -o libfawnds_la-DOMTreeErrorReporter.lo `test -f 'DOMTreeErrorReporter.cpp' || echo '$(srcdir)/'`DOMTreeErrorReporter.cpp +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-DOMTreeErrorReporter.Tpo $(DEPDIR)/libfawnds_la-DOMTreeErrorReporter.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='DOMTreeErrorReporter.cpp' object='libfawnds_la-DOMTreeErrorReporter.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-DOMTreeErrorReporter.lo `test -f 'DOMTreeErrorReporter.cpp' || echo '$(srcdir)/'`DOMTreeErrorReporter.cpp + +libfawnds_la-DOMWriteErrorHandler.lo: DOMWriteErrorHandler.cpp +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-DOMWriteErrorHandler.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-DOMWriteErrorHandler.Tpo -c -o libfawnds_la-DOMWriteErrorHandler.lo `test -f 'DOMWriteErrorHandler.cpp' || echo '$(srcdir)/'`DOMWriteErrorHandler.cpp +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-DOMWriteErrorHandler.Tpo $(DEPDIR)/libfawnds_la-DOMWriteErrorHandler.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='DOMWriteErrorHandler.cpp' object='libfawnds_la-DOMWriteErrorHandler.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-DOMWriteErrorHandler.lo `test -f 'DOMWriteErrorHandler.cpp' || echo '$(srcdir)/'`DOMWriteErrorHandler.cpp + +libfawnds_la-configuration.lo: configuration.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-configuration.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-configuration.Tpo -c -o libfawnds_la-configuration.lo `test -f 'configuration.cc' || echo '$(srcdir)/'`configuration.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-configuration.Tpo $(DEPDIR)/libfawnds_la-configuration.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='configuration.cc' object='libfawnds_la-configuration.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-configuration.lo `test -f 'configuration.cc' || echo '$(srcdir)/'`configuration.cc + +libfawnds_la-file_io.lo: file_io.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-file_io.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-file_io.Tpo -c -o libfawnds_la-file_io.lo `test -f 'file_io.cc' || echo '$(srcdir)/'`file_io.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-file_io.Tpo $(DEPDIR)/libfawnds_la-file_io.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='file_io.cc' object='libfawnds_la-file_io.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-file_io.lo `test -f 'file_io.cc' || echo '$(srcdir)/'`file_io.cc + +libfawnds_la-task.lo: task.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-task.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-task.Tpo -c -o libfawnds_la-task.lo `test -f 'task.cc' || echo '$(srcdir)/'`task.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-task.Tpo $(DEPDIR)/libfawnds_la-task.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='task.cc' object='libfawnds_la-task.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-task.lo `test -f 'task.cc' || echo '$(srcdir)/'`task.cc + +libfawnds_la-rate_limiter.lo: rate_limiter.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-rate_limiter.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-rate_limiter.Tpo -c -o libfawnds_la-rate_limiter.lo `test -f 'rate_limiter.cc' || echo '$(srcdir)/'`rate_limiter.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-rate_limiter.Tpo $(DEPDIR)/libfawnds_la-rate_limiter.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='rate_limiter.cc' object='libfawnds_la-rate_limiter.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-rate_limiter.lo `test -f 'rate_limiter.cc' || echo '$(srcdir)/'`rate_limiter.cc + +libfawnds_la-global_limits.lo: global_limits.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-global_limits.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-global_limits.Tpo -c -o libfawnds_la-global_limits.lo `test -f 'global_limits.cc' || echo '$(srcdir)/'`global_limits.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-global_limits.Tpo $(DEPDIR)/libfawnds_la-global_limits.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='global_limits.cc' object='libfawnds_la-global_limits.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-global_limits.lo `test -f 'global_limits.cc' || echo '$(srcdir)/'`global_limits.cc + +libfawnds_la-fawnds_iterator.lo: fawnds_iterator.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-fawnds_iterator.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-fawnds_iterator.Tpo -c -o libfawnds_la-fawnds_iterator.lo `test -f 'fawnds_iterator.cc' || echo '$(srcdir)/'`fawnds_iterator.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-fawnds_iterator.Tpo $(DEPDIR)/libfawnds_la-fawnds_iterator.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='fawnds_iterator.cc' object='libfawnds_la-fawnds_iterator.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-fawnds_iterator.lo `test -f 'fawnds_iterator.cc' || echo '$(srcdir)/'`fawnds_iterator.cc + +libfawnds_la-fawnds.lo: fawnds.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-fawnds.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-fawnds.Tpo -c -o libfawnds_la-fawnds.lo `test -f 'fawnds.cc' || echo '$(srcdir)/'`fawnds.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-fawnds.Tpo $(DEPDIR)/libfawnds_la-fawnds.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='fawnds.cc' object='libfawnds_la-fawnds.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-fawnds.lo `test -f 'fawnds.cc' || echo '$(srcdir)/'`fawnds.cc + +libfawnds_la-fawnds_factory.lo: fawnds_factory.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-fawnds_factory.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-fawnds_factory.Tpo -c -o libfawnds_la-fawnds_factory.lo `test -f 'fawnds_factory.cc' || echo '$(srcdir)/'`fawnds_factory.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-fawnds_factory.Tpo $(DEPDIR)/libfawnds_la-fawnds_factory.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='fawnds_factory.cc' object='libfawnds_la-fawnds_factory.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-fawnds_factory.lo `test -f 'fawnds_factory.cc' || echo '$(srcdir)/'`fawnds_factory.cc + +libfawnds_la-hash_functions.lo: hash_functions.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-hash_functions.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-hash_functions.Tpo -c -o libfawnds_la-hash_functions.lo `test -f 'hash_functions.cc' || echo '$(srcdir)/'`hash_functions.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-hash_functions.Tpo $(DEPDIR)/libfawnds_la-hash_functions.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='hash_functions.cc' object='libfawnds_la-hash_functions.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-hash_functions.lo `test -f 'hash_functions.cc' || echo '$(srcdir)/'`hash_functions.cc + +libfawnds_la-hash_table_default.lo: hash_table_default.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-hash_table_default.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-hash_table_default.Tpo -c -o libfawnds_la-hash_table_default.lo `test -f 'hash_table_default.cc' || echo '$(srcdir)/'`hash_table_default.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-hash_table_default.Tpo $(DEPDIR)/libfawnds_la-hash_table_default.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='hash_table_default.cc' object='libfawnds_la-hash_table_default.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-hash_table_default.lo `test -f 'hash_table_default.cc' || echo '$(srcdir)/'`hash_table_default.cc + +libfawnds_la-hash_table_cuckoo.lo: hash_table_cuckoo.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-hash_table_cuckoo.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-hash_table_cuckoo.Tpo -c -o libfawnds_la-hash_table_cuckoo.lo `test -f 'hash_table_cuckoo.cc' || echo '$(srcdir)/'`hash_table_cuckoo.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-hash_table_cuckoo.Tpo $(DEPDIR)/libfawnds_la-hash_table_cuckoo.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='hash_table_cuckoo.cc' object='libfawnds_la-hash_table_cuckoo.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-hash_table_cuckoo.lo `test -f 'hash_table_cuckoo.cc' || echo '$(srcdir)/'`hash_table_cuckoo.cc + +libfawnds_la-file_store.lo: file_store.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-file_store.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-file_store.Tpo -c -o libfawnds_la-file_store.lo `test -f 'file_store.cc' || echo '$(srcdir)/'`file_store.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-file_store.Tpo $(DEPDIR)/libfawnds_la-file_store.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='file_store.cc' object='libfawnds_la-file_store.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-file_store.lo `test -f 'file_store.cc' || echo '$(srcdir)/'`file_store.cc + +libfawnds_la-sorter.lo: sorter.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-sorter.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-sorter.Tpo -c -o libfawnds_la-sorter.lo `test -f 'sorter.cc' || echo '$(srcdir)/'`sorter.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-sorter.Tpo $(DEPDIR)/libfawnds_la-sorter.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='sorter.cc' object='libfawnds_la-sorter.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-sorter.lo `test -f 'sorter.cc' || echo '$(srcdir)/'`sorter.cc + +libfawnds_la-fawnds_partition.lo: fawnds_partition.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-fawnds_partition.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-fawnds_partition.Tpo -c -o libfawnds_la-fawnds_partition.lo `test -f 'fawnds_partition.cc' || echo '$(srcdir)/'`fawnds_partition.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-fawnds_partition.Tpo $(DEPDIR)/libfawnds_la-fawnds_partition.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='fawnds_partition.cc' object='libfawnds_la-fawnds_partition.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-fawnds_partition.lo `test -f 'fawnds_partition.cc' || echo '$(srcdir)/'`fawnds_partition.cc + +libfawnds_la-fawnds_combi.lo: fawnds_combi.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-fawnds_combi.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-fawnds_combi.Tpo -c -o libfawnds_la-fawnds_combi.lo `test -f 'fawnds_combi.cc' || echo '$(srcdir)/'`fawnds_combi.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-fawnds_combi.Tpo $(DEPDIR)/libfawnds_la-fawnds_combi.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='fawnds_combi.cc' object='libfawnds_la-fawnds_combi.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-fawnds_combi.lo `test -f 'fawnds_combi.cc' || echo '$(srcdir)/'`fawnds_combi.cc + +libfawnds_la-fawnds_sf.lo: fawnds_sf.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-fawnds_sf.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-fawnds_sf.Tpo -c -o libfawnds_la-fawnds_sf.lo `test -f 'fawnds_sf.cc' || echo '$(srcdir)/'`fawnds_sf.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-fawnds_sf.Tpo $(DEPDIR)/libfawnds_la-fawnds_sf.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='fawnds_sf.cc' object='libfawnds_la-fawnds_sf.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-fawnds_sf.lo `test -f 'fawnds_sf.cc' || echo '$(srcdir)/'`fawnds_sf.cc + +libfawnds_la-fawnds_sf_ordered_trie.lo: fawnds_sf_ordered_trie.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-fawnds_sf_ordered_trie.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-fawnds_sf_ordered_trie.Tpo -c -o libfawnds_la-fawnds_sf_ordered_trie.lo `test -f 'fawnds_sf_ordered_trie.cc' || echo '$(srcdir)/'`fawnds_sf_ordered_trie.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-fawnds_sf_ordered_trie.Tpo $(DEPDIR)/libfawnds_la-fawnds_sf_ordered_trie.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='fawnds_sf_ordered_trie.cc' object='libfawnds_la-fawnds_sf_ordered_trie.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-fawnds_sf_ordered_trie.lo `test -f 'fawnds_sf_ordered_trie.cc' || echo '$(srcdir)/'`fawnds_sf_ordered_trie.cc + +libfawnds_la-fawnds_proxy.lo: fawnds_proxy.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-fawnds_proxy.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-fawnds_proxy.Tpo -c -o libfawnds_la-fawnds_proxy.lo `test -f 'fawnds_proxy.cc' || echo '$(srcdir)/'`fawnds_proxy.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-fawnds_proxy.Tpo $(DEPDIR)/libfawnds_la-fawnds_proxy.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='fawnds_proxy.cc' object='libfawnds_la-fawnds_proxy.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-fawnds_proxy.lo `test -f 'fawnds_proxy.cc' || echo '$(srcdir)/'`fawnds_proxy.cc + +libfawnds_la-fawnds_monitor.lo: fawnds_monitor.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-fawnds_monitor.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-fawnds_monitor.Tpo -c -o libfawnds_la-fawnds_monitor.lo `test -f 'fawnds_monitor.cc' || echo '$(srcdir)/'`fawnds_monitor.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-fawnds_monitor.Tpo $(DEPDIR)/libfawnds_la-fawnds_monitor.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='fawnds_monitor.cc' object='libfawnds_la-fawnds_monitor.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-fawnds_monitor.lo `test -f 'fawnds_monitor.cc' || echo '$(srcdir)/'`fawnds_monitor.cc + +libfawnds_la-bdb.lo: bdb.cc +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-bdb.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-bdb.Tpo -c -o libfawnds_la-bdb.lo `test -f 'bdb.cc' || echo '$(srcdir)/'`bdb.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-bdb.Tpo $(DEPDIR)/libfawnds_la-bdb.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='bdb.cc' object='libfawnds_la-bdb.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-bdb.lo `test -f 'bdb.cc' || echo '$(srcdir)/'`bdb.cc + +libfawnds_la-bit_access.lo: cindex/bit_access.cpp +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-bit_access.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-bit_access.Tpo -c -o libfawnds_la-bit_access.lo `test -f 'cindex/bit_access.cpp' || echo '$(srcdir)/'`cindex/bit_access.cpp +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-bit_access.Tpo $(DEPDIR)/libfawnds_la-bit_access.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='cindex/bit_access.cpp' object='libfawnds_la-bit_access.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-bit_access.lo `test -f 'cindex/bit_access.cpp' || echo '$(srcdir)/'`cindex/bit_access.cpp + +libfawnds_la-bit_vector.lo: cindex/bit_vector.cpp +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-bit_vector.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-bit_vector.Tpo -c -o libfawnds_la-bit_vector.lo `test -f 'cindex/bit_vector.cpp' || echo '$(srcdir)/'`cindex/bit_vector.cpp +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-bit_vector.Tpo $(DEPDIR)/libfawnds_la-bit_vector.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='cindex/bit_vector.cpp' object='libfawnds_la-bit_vector.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-bit_vector.lo `test -f 'cindex/bit_vector.cpp' || echo '$(srcdir)/'`cindex/bit_vector.cpp + +libfawnds_la-expected_size.lo: cindex/expected_size.cpp +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-expected_size.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-expected_size.Tpo -c -o libfawnds_la-expected_size.lo `test -f 'cindex/expected_size.cpp' || echo '$(srcdir)/'`cindex/expected_size.cpp +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-expected_size.Tpo $(DEPDIR)/libfawnds_la-expected_size.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='cindex/expected_size.cpp' object='libfawnds_la-expected_size.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-expected_size.lo `test -f 'cindex/expected_size.cpp' || echo '$(srcdir)/'`cindex/expected_size.cpp + +libfawnds_la-flat_absoff_bucketing.lo: cindex/flat_absoff_bucketing.cpp +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-flat_absoff_bucketing.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-flat_absoff_bucketing.Tpo -c -o libfawnds_la-flat_absoff_bucketing.lo `test -f 'cindex/flat_absoff_bucketing.cpp' || echo '$(srcdir)/'`cindex/flat_absoff_bucketing.cpp +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-flat_absoff_bucketing.Tpo $(DEPDIR)/libfawnds_la-flat_absoff_bucketing.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='cindex/flat_absoff_bucketing.cpp' object='libfawnds_la-flat_absoff_bucketing.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-flat_absoff_bucketing.lo `test -f 'cindex/flat_absoff_bucketing.cpp' || echo '$(srcdir)/'`cindex/flat_absoff_bucketing.cpp + +libfawnds_la-twolevel_absoff_bucketing.lo: cindex/twolevel_absoff_bucketing.cpp +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-twolevel_absoff_bucketing.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-twolevel_absoff_bucketing.Tpo -c -o libfawnds_la-twolevel_absoff_bucketing.lo `test -f 'cindex/twolevel_absoff_bucketing.cpp' || echo '$(srcdir)/'`cindex/twolevel_absoff_bucketing.cpp +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-twolevel_absoff_bucketing.Tpo $(DEPDIR)/libfawnds_la-twolevel_absoff_bucketing.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='cindex/twolevel_absoff_bucketing.cpp' object='libfawnds_la-twolevel_absoff_bucketing.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-twolevel_absoff_bucketing.lo `test -f 'cindex/twolevel_absoff_bucketing.cpp' || echo '$(srcdir)/'`cindex/twolevel_absoff_bucketing.cpp + +libfawnds_la-twolevel_reloff_bucketing.lo: cindex/twolevel_reloff_bucketing.cpp +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-twolevel_reloff_bucketing.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-twolevel_reloff_bucketing.Tpo -c -o libfawnds_la-twolevel_reloff_bucketing.lo `test -f 'cindex/twolevel_reloff_bucketing.cpp' || echo '$(srcdir)/'`cindex/twolevel_reloff_bucketing.cpp +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-twolevel_reloff_bucketing.Tpo $(DEPDIR)/libfawnds_la-twolevel_reloff_bucketing.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='cindex/twolevel_reloff_bucketing.cpp' object='libfawnds_la-twolevel_reloff_bucketing.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-twolevel_reloff_bucketing.lo `test -f 'cindex/twolevel_reloff_bucketing.cpp' || echo '$(srcdir)/'`cindex/twolevel_reloff_bucketing.cpp + +libfawnds_la-semi_direct_16_absoff_bucketing.lo: cindex/semi_direct_16_absoff_bucketing.cpp +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-semi_direct_16_absoff_bucketing.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-semi_direct_16_absoff_bucketing.Tpo -c -o libfawnds_la-semi_direct_16_absoff_bucketing.lo `test -f 'cindex/semi_direct_16_absoff_bucketing.cpp' || echo '$(srcdir)/'`cindex/semi_direct_16_absoff_bucketing.cpp +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-semi_direct_16_absoff_bucketing.Tpo $(DEPDIR)/libfawnds_la-semi_direct_16_absoff_bucketing.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='cindex/semi_direct_16_absoff_bucketing.cpp' object='libfawnds_la-semi_direct_16_absoff_bucketing.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-semi_direct_16_absoff_bucketing.lo `test -f 'cindex/semi_direct_16_absoff_bucketing.cpp' || echo '$(srcdir)/'`cindex/semi_direct_16_absoff_bucketing.cpp + +libfawnds_la-semi_direct_16_reloff_bucketing.lo: cindex/semi_direct_16_reloff_bucketing.cpp +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-semi_direct_16_reloff_bucketing.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-semi_direct_16_reloff_bucketing.Tpo -c -o libfawnds_la-semi_direct_16_reloff_bucketing.lo `test -f 'cindex/semi_direct_16_reloff_bucketing.cpp' || echo '$(srcdir)/'`cindex/semi_direct_16_reloff_bucketing.cpp +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-semi_direct_16_reloff_bucketing.Tpo $(DEPDIR)/libfawnds_la-semi_direct_16_reloff_bucketing.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='cindex/semi_direct_16_reloff_bucketing.cpp' object='libfawnds_la-semi_direct_16_reloff_bucketing.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-semi_direct_16_reloff_bucketing.lo `test -f 'cindex/semi_direct_16_reloff_bucketing.cpp' || echo '$(srcdir)/'`cindex/semi_direct_16_reloff_bucketing.cpp + +libfawnds_la-bucketing_index.lo: cindex/bucketing_index.cpp +@am__fastdepCXX_TRUE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libfawnds_la-bucketing_index.lo -MD -MP -MF $(DEPDIR)/libfawnds_la-bucketing_index.Tpo -c -o libfawnds_la-bucketing_index.lo `test -f 'cindex/bucketing_index.cpp' || echo '$(srcdir)/'`cindex/bucketing_index.cpp +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/libfawnds_la-bucketing_index.Tpo $(DEPDIR)/libfawnds_la-bucketing_index.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='cindex/bucketing_index.cpp' object='libfawnds_la-bucketing_index.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libfawnds_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libfawnds_la-bucketing_index.lo `test -f 'cindex/bucketing_index.cpp' || echo '$(srcdir)/'`cindex/bucketing_index.cpp + +fawnds_bench-fawnds_bench.o: fawnds_bench.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(fawnds_bench_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT fawnds_bench-fawnds_bench.o -MD -MP -MF $(DEPDIR)/fawnds_bench-fawnds_bench.Tpo -c -o fawnds_bench-fawnds_bench.o `test -f 'fawnds_bench.cc' || echo '$(srcdir)/'`fawnds_bench.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/fawnds_bench-fawnds_bench.Tpo $(DEPDIR)/fawnds_bench-fawnds_bench.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='fawnds_bench.cc' object='fawnds_bench-fawnds_bench.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(fawnds_bench_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o fawnds_bench-fawnds_bench.o `test -f 'fawnds_bench.cc' || echo '$(srcdir)/'`fawnds_bench.cc + +fawnds_bench-fawnds_bench.obj: fawnds_bench.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(fawnds_bench_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT fawnds_bench-fawnds_bench.obj -MD -MP -MF $(DEPDIR)/fawnds_bench-fawnds_bench.Tpo -c -o fawnds_bench-fawnds_bench.obj `if test -f 'fawnds_bench.cc'; then $(CYGPATH_W) 'fawnds_bench.cc'; else $(CYGPATH_W) '$(srcdir)/fawnds_bench.cc'; fi` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/fawnds_bench-fawnds_bench.Tpo $(DEPDIR)/fawnds_bench-fawnds_bench.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='fawnds_bench.cc' object='fawnds_bench-fawnds_bench.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(fawnds_bench_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o fawnds_bench-fawnds_bench.obj `if test -f 'fawnds_bench.cc'; then $(CYGPATH_W) 'fawnds_bench.cc'; else $(CYGPATH_W) '$(srcdir)/fawnds_bench.cc'; fi` + +.cpp.o: +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ $< + +.cpp.obj: +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.cpp.lo: +@am__fastdepCXX_TRUE@ $(LTCXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + mkid -fID $$unique +tags: TAGS + +TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + set x; \ + here=`pwd`; \ + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: CTAGS +CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(PROGRAMS) $(HEADERS) +installdirs: +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + `test -z '$(STRIP)' || \ + echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install +mostlyclean-generic: + +clean-generic: + -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES) + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS mostlyclean-am + +distclean: distclean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \ + clean-libtool clean-noinstLTLIBRARIES clean-noinstPROGRAMS \ + ctags distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-data \ + install-data-am install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-man install-pdf install-pdf-am \ + install-ps install-ps-am install-strip installcheck \ + installcheck-am installdirs maintainer-clean \ + maintainer-clean-generic mostlyclean mostlyclean-compile \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags uninstall uninstall-am + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/fawnds/basic_types.h b/fawnds/basic_types.h new file mode 100644 index 0000000..e0102df --- /dev/null +++ b/fawnds/basic_types.h @@ -0,0 +1,19 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _BASIC_TYPES_H_ +#define _BASIC_TYPES_H_ + +// include essential headers only -- e.g. type definition common to all classes in FawnDS + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include +#include // using cstdint may show C++0x error +#include // using cinttypes may show C++0x error + +#include + +#include + +#endif // #ifndef _BASIC_TYPES_H_ diff --git a/fawnds/bdb.cc b/fawnds/bdb.cc new file mode 100644 index 0000000..569f52c --- /dev/null +++ b/fawnds/bdb.cc @@ -0,0 +1,561 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "bdb.h" +#include "configuration.h" +#include +#include + +#ifdef HAVE_LIBDB + +namespace fawn { + + BDB::BDB() + : dbp_(NULL) + { + } + + BDB::~BDB() + { + if (dbp_) + Close(); + } + + FawnDS_Return + BDB::Create() + { + if (dbp_) + return ERROR; + + // TODO: use DB_ENV to optimize BDB for SSD? + + int ret = db_create(&dbp_, NULL, 0); + if (ret != 0) { + fprintf(stderr, "BDB::Create(): Error while creating DB: %s\n", db_strerror(ret)); + return ERROR; + } + + std::string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + + ret = dbp_->open(dbp_, NULL, filename.c_str(), NULL, DB_BTREE, DB_THREAD | DB_CREATE | DB_TRUNCATE, 0); + if (ret != 0) { + fprintf(stderr, "BDB::Create(): Error while creating DB: %s\n", db_strerror(ret)); + dbp_->close(dbp_, 0); + dbp_ = NULL; + return ERROR; + } + + size_ = 0; + + return OK; + } + + FawnDS_Return + BDB::Open() + { + if (dbp_) + return ERROR; + + int ret = db_create(&dbp_, NULL, 0); + if (ret != 0) { + fprintf(stderr, "BDB::Open(): Error while creating DB: %s\n", db_strerror(ret)); + return ERROR; + } + + std::string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + + ret = dbp_->open(dbp_, NULL, filename.c_str(), NULL, DB_BTREE, DB_THREAD, 0); + if (ret != 0) { + fprintf(stderr, "BDB::Open(): Error while creating DB: %s\n", db_strerror(ret)); + dbp_->close(dbp_, 0); + dbp_ = NULL; + return ERROR; + } + + // TODO: load size_ from file + size_ = 0; + + return OK; + } + + FawnDS_Return + BDB::ConvertTo(FawnDS* new_store) const + { + BDB* bdb = dynamic_cast(new_store); + if (!bdb) + return UNSUPPORTED; + + bdb->Close(); + + std::string src_filename = config_->GetStringValue("child::file") + "_"; + src_filename += config_->GetStringValue("child::id"); + + std::string dest_filename = bdb->config_->GetStringValue("child::file") + "_"; + dest_filename += bdb->config_->GetStringValue("child::id"); + + if (unlink(dest_filename.c_str())) { + fprintf(stderr, "BDB::ConvertTo(): cannot unlink the destination file\n"); + } + if (link(src_filename.c_str(), dest_filename.c_str())) { + fprintf(stderr, "BDB::ConvertTo(): cannot link the destination file\n"); + } + + if (bdb->Open() != OK) + return ERROR; + + return OK; + } + + FawnDS_Return + BDB::Flush() + { + if (!dbp_) + return ERROR; + + // TODO: implement + return OK; + } + + FawnDS_Return + BDB::Close() + { + if (!dbp_) + return ERROR; + + dbp_->close(dbp_, 0); + dbp_ = NULL; + + return OK; + } + + FawnDS_Return + BDB::Destroy() + { + if (dbp_) + return ERROR; + + DB* dbp; + + int ret = db_create(&dbp, NULL, 0); + if (ret != 0) { + fprintf(stderr, "BDB::Destroy(): Error while creating DB: %s\n", db_strerror(ret)); + return ERROR; + } + + std::string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + + ret = dbp->remove(dbp, filename.c_str(), NULL, 0); + if (ret != 0) { + fprintf(stderr, "BDB::Destroy(): Error while removing DB: %s\n", db_strerror(ret)); + dbp->close(dbp, 0); + return ERROR; + } + + dbp->close(dbp, 0); + return OK; + } + + FawnDS_Return + BDB::Status(const FawnDS_StatusType& type, Value& status) const + { + if (!dbp_) + return ERROR; + + std::ostringstream oss; + switch (type) { + case NUM_DATA: + case NUM_ACTIVE_DATA: + oss << size_; + break; + case CAPACITY: + oss << -1; // unlimited + break; + case MEMORY_USE: + { + uint32_t gbytes = 0; + uint32_t bytes = 0; + int ncache = 0; + int ret = dbp_->get_cachesize(dbp_, &gbytes, &bytes, &ncache); + if (ret != 0) { + fprintf(stderr, "BDB::Status(): Error while querying DB: %s\n", db_strerror(ret)); + return ERROR; + } + oss << gbytes * (1024 * 1024 * 1024) + bytes; // BDB uses powers of two + } + break; + case DISK_USE: + { + // TODO: check if this work + DB_BTREE_STAT stat; + int ret = dbp_->stat(dbp_, NULL, &stat, DB_FAST_STAT); + if (ret != 0) { + fprintf(stderr, "BDB::Status(): Error while querying DB: %s\n", db_strerror(ret)); + return ERROR; + } + oss << stat.bt_pagecnt * stat.bt_pagesize; + } + break; + default: + return UNSUPPORTED; + } + status = NewValue(oss.str()); + return OK; + } + + FawnDS_Return + BDB::Put(const ConstValue& key, const ConstValue& data) + { + if (!dbp_) + return ERROR; + + if (key.size() == 0) + return INVALID_KEY; + + DBT key_v; + memset(&key_v, 0, sizeof(DBT)); + key_v.data = const_cast(key.data()); + key_v.size = key.size(); + + DBT data_v; + memset(&data_v, 0, sizeof(DBT)); + data_v.data = const_cast(data.data()); + data_v.size = data.size(); + + int ret = dbp_->put(dbp_, NULL, &key_v, &data_v, DB_NOOVERWRITE); + if (ret == 0) { + ++size_; + return OK; + } + else if (ret == DB_KEYEXIST) + ret = dbp_->put(dbp_, NULL, &key_v, &data_v, 0); + + if (ret != 0) { + fprintf(stderr, "BDB::Put(): %s\n", db_strerror(ret)); + return ERROR; + } + + return OK; + } + + FawnDS_Return + BDB::Delete(const ConstValue& key) + { + if (!dbp_) + return ERROR; + + if (key.size() == 0) + return INVALID_KEY; + + DBT key_v; + memset(&key_v, 0, sizeof(DBT)); + key_v.data = const_cast(key.data()); + key_v.size = key.size(); + + int ret = dbp_->del(dbp_, NULL, &key_v, 0); + if (ret == 0) { + --size_; + return OK; + } + else if (ret == DB_NOTFOUND) + return KEY_NOT_FOUND; + if (ret != 0) { + fprintf(stderr, "BDB::Delete(): %s\n", db_strerror(ret)); + return ERROR; + } + + return OK; + } + + FawnDS_Return + BDB::Contains(const ConstValue& key) const + { + if (!dbp_) + return ERROR; + + if (key.size() == 0) + return INVALID_KEY; + + DBT key_v; + memset(&key_v, 0, sizeof(DBT)); + key_v.data = const_cast(key.data()); + key_v.size = key.size(); + + int ret = dbp_->exists(dbp_, NULL, &key_v, 0); + if (ret == 0) + return OK; + else if (ret == DB_NOTFOUND) + return KEY_NOT_FOUND; + else { + fprintf(stderr, "BDB::Contains(): %s\n", db_strerror(ret)); + return ERROR; + } + } + + FawnDS_Return + BDB::Length(const ConstValue& key, size_t& len) const + { + if (!dbp_) + return ERROR; + + if (key.size() == 0) + return INVALID_KEY; + + DBT key_v; + memset(&key_v, 0, sizeof(DBT)); + key_v.data = const_cast(key.data()); + key_v.size = key.size(); + + DBT data_v; + memset(&data_v, 0, sizeof(DBT)); + data_v.flags = DB_DBT_USERMEM; + + int ret = dbp_->get(dbp_, NULL, &key_v, &data_v, 0); + if (ret == 0 || ret == DB_BUFFER_SMALL) { + len = data_v.size; + return OK; + } + else if (ret == DB_NOTFOUND) + return KEY_NOT_FOUND; + else { + fprintf(stderr, "BDB::Length(): %s\n", db_strerror(ret)); + return ERROR; + } + } + + FawnDS_Return + BDB::Get(const ConstValue& key, Value& data, size_t offset, size_t len) const + { + if (!dbp_) + return ERROR; + + if (key.size() == 0) + return INVALID_KEY; + + size_t data_len = 0; + FawnDS_Return ret_len = Length(key, data_len); + if (ret_len != OK) + return ret_len; + + if (offset > data_len) + return END; + + if (offset + len > data_len) + len = data_len - offset; + + data.resize(len, false); + + DBT key_v; + memset(&key_v, 0, sizeof(DBT)); + key_v.data = const_cast(key.data()); + key_v.size = key.size(); + + DBT data_v; + memset(&data_v, 0, sizeof(DBT)); + data_v.data = data.data(); + data_v.ulen = len; + data_v.flags = DB_DBT_USERMEM; + + int ret = dbp_->get(dbp_, NULL, &key_v, &data_v, 0); + if (ret != 0) { + fprintf(stderr, "BDB::Get(): %s\n", db_strerror(ret)); + return ERROR; + } + return OK; + } + + FawnDS_ConstIterator + BDB::Enumerate() const + { + if (!dbp_) + return FawnDS_ConstIterator(); + + IteratorElem* elem = new IteratorElem(this); + + int ret = dbp_->cursor(dbp_, NULL, &elem->cursor, 0); + if (ret != 0) { + fprintf(stderr, "BDB::Enumerate(): %s\n", db_strerror(ret)); + return FawnDS_ConstIterator(); + } + + elem->Increment(true); + + return FawnDS_ConstIterator(elem); + } + + FawnDS_Iterator + BDB::Enumerate() + { + if (!dbp_) + return FawnDS_Iterator(); + + IteratorElem* elem = new IteratorElem(this); + + int ret = dbp_->cursor(dbp_, NULL, &elem->cursor, 0); + if (ret != 0) { + fprintf(stderr, "BDB::Enumerate(): %s\n", db_strerror(ret)); + return FawnDS_Iterator(); + } + + elem->Increment(true); + + return FawnDS_Iterator(elem); + } + + FawnDS_ConstIterator + BDB::Find(const ConstValue& key) const + { + if (!dbp_) + return FawnDS_ConstIterator(); + + if (key.size() == 0) + return FawnDS_ConstIterator(); + + IteratorElem* elem = new IteratorElem(this); + + elem->key = key; + + int ret = dbp_->cursor(dbp_, NULL, &elem->cursor, 0); + if (ret != 0) { + fprintf(stderr, "BDB::Find(): %s\n", db_strerror(ret)); + delete elem; + return FawnDS_ConstIterator(); + } + + elem->Increment(true); + + return FawnDS_ConstIterator(elem); + } + + FawnDS_Iterator + BDB::Find(const ConstValue& key) + { + if (!dbp_) + return FawnDS_Iterator(); + + if (key.size() == 0) + return FawnDS_Iterator(); + + IteratorElem* elem = new IteratorElem(this); + + elem->key = key; + + int ret = dbp_->cursor(dbp_, NULL, &elem->cursor, 0); + if (ret != 0) { + fprintf(stderr, "BDB::Find(): %s\n", db_strerror(ret)); + delete elem; + return FawnDS_Iterator(); + } + + elem->Increment(true); + + return FawnDS_Iterator(elem); + } + + BDB::IteratorElem::IteratorElem(const BDB* fawnds) + { + this->fawnds = fawnds; + } + + BDB::IteratorElem::~IteratorElem() + { + cursor->close(cursor); + cursor = NULL; + } + + FawnDS_IteratorElem* + BDB::IteratorElem::Clone() const + { + IteratorElem* elem = new IteratorElem(static_cast(fawnds)); + *elem = *this; + int ret = cursor->dup(cursor, &elem->cursor, DB_POSITION); + if (ret != 0) { + fprintf(stderr, "BDB::Clone(): %s\n", db_strerror(ret)); + return NULL; + } + return elem; + } + + void + BDB::IteratorElem::Next() + { + Increment(false); + } + + void + BDB::IteratorElem::Increment(bool initial) + { + int flags; + + DBT key_v; + memset(&key_v, 0, sizeof(DBT)); + DBT data_v; + memset(&data_v, 0, sizeof(DBT)); + + if (initial) { + if (key.size() == 0) { + flags = DB_FIRST; + key_v.flags = DB_DBT_USERMEM; + data_v.flags = DB_DBT_USERMEM; + } + else { + flags = DB_SET; + key_v.data = key.data(); + key_v.size = key.size(); + data_v.flags = DB_DBT_USERMEM; + } + } + else { + key_v.flags = DB_DBT_USERMEM; + data_v.flags = DB_DBT_USERMEM; + flags = DB_NEXT; + } + + // obtain the length of key & data + int ret = cursor->get(cursor, &key_v, &data_v, flags); + if (ret == 0) { + // this should not happen because we have key_v.ulen = 0, and there is no zero-length key + assert(false); + state = END; + return; + } + else if (ret == DB_NOTFOUND) { + state = END; + return; + } + else if (ret != DB_BUFFER_SMALL) { + fprintf(stderr, "BDB::IteratorElem::Increment(): Error while obtaining length: %s\n", db_strerror(ret)); + state = END; + return; + } + + //fprintf(stderr, "%d %d\n", key_v.size, data_v.size); + + // retrieve key & data + key.resize(key_v.size, false); + key_v.data = key.data(); + key_v.size = key.size(); + key_v.ulen = key_v.size; + key_v.flags = DB_DBT_USERMEM; + + data.resize(data_v.size, false); + data_v.data = data.data(); + data_v.size = data.size(); + data_v.ulen = data_v.size; + key_v.flags = DB_DBT_USERMEM; + + ret = cursor->get(cursor, &key_v, &data_v, flags); + if (ret != 0) { + fprintf(stderr, "BDB::IteratorElem::Increment(): Error while reading key and data: %s\n", db_strerror(ret)); + state = END; + return; + } + + state = OK; + key.resize(key_v.size); + data.resize(data_v.size); + } + +} // namespace fawn + +#endif diff --git a/fawnds/bdb.h b/fawnds/bdb.h new file mode 100644 index 0000000..5422737 --- /dev/null +++ b/fawnds/bdb.h @@ -0,0 +1,78 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _BDB_H_ +#define _BDB_H_ + +#include "fawnds.h" +#include "config.h" +#include + +#ifdef HAVE_LIBDB +#include + +namespace fawn { + + // configuration + // : "bdb" (fixed) + // : the ID of the store + // : the file name prefix of Berkeley Database + + class BDB : public FawnDS { + public: + BDB(); + virtual ~BDB(); + + virtual FawnDS_Return Create(); + virtual FawnDS_Return Open(); + + virtual FawnDS_Return ConvertTo(FawnDS* new_store) const; + + virtual FawnDS_Return Flush(); + virtual FawnDS_Return Close(); + + virtual FawnDS_Return Destroy(); + + virtual FawnDS_Return Status(const FawnDS_StatusType& type, Value& status) const; + + virtual FawnDS_Return Put(const ConstValue& key, const ConstValue& data); + //virtual FawnDS_Return Append(Value& key, const ConstValue& data); + + virtual FawnDS_Return Delete(const ConstValue& key); + + virtual FawnDS_Return Contains(const ConstValue& key) const; + virtual FawnDS_Return Length(const ConstValue& key, size_t& len) const; + virtual FawnDS_Return Get(const ConstValue& key, Value& data, size_t offset = 0, size_t len = -1) const; + + virtual FawnDS_ConstIterator Enumerate() const; + virtual FawnDS_Iterator Enumerate(); + + virtual FawnDS_ConstIterator Find(const ConstValue& key) const; + virtual FawnDS_Iterator Find(const ConstValue& key); + + struct IteratorElem : public FawnDS_IteratorElem { + IteratorElem(const BDB* fawnds); + ~IteratorElem(); + + FawnDS_IteratorElem* Clone() const; + void Next(); + + void Increment(bool initial); + + DBC* cursor; + }; + + private: + DB* dbp_; + + size_t key_len_; + size_t data_len_; + + tbb::atomic size_; + + friend struct IteratorElem; + }; + +} // namespace fawn + +#endif + +#endif // #ifndef _FAWNDS_SF_H_ diff --git a/fawnds/cindex/basic_types.hpp b/fawnds/cindex/basic_types.hpp new file mode 100644 index 0000000..228c596 --- /dev/null +++ b/fawnds/cindex/basic_types.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include "common.hpp" + +#include +#include + +namespace cindex +{ + // guarded static_cast + template + inline T guarded_cast(const S& v) + { +#ifndef NDEBUG + return boost::numeric::converter::convert(v); +#else + return static_cast(v); +#endif + } +} + diff --git a/fawnds/cindex/bit_access.cpp b/fawnds/cindex/bit_access.cpp new file mode 100644 index 0000000..b119889 --- /dev/null +++ b/fawnds/cindex/bit_access.cpp @@ -0,0 +1,47 @@ +#include "bit_access.hpp" + +namespace cindex +{ + /* +#define CINDEX_SHIFT(base) \ + 1ull << ((base) + 7), 1ull << ((base) + 6), 1ull << ((base) + 5), 1ull << ((base) + 4), 1ull << ((base) + 3), 1ull << ((base) + 2), 1ull << ((base) + 1), 1ull << ((base) + 0), + + template<> + const uint8_t bit_table::bit_table_[block_info::bits_per_block] = + { + CINDEX_SHIFT(0) + }; + + template<> + const uint16_t bit_table::bit_table_[block_info::bits_per_block] = + { + CINDEX_SHIFT(8) + CINDEX_SHIFT(0) + }; + + template<> + const uint32_t bit_table::bit_table_[block_info::bits_per_block] = + { + CINDEX_SHIFT(24) + CINDEX_SHIFT(16) + CINDEX_SHIFT(8) + CINDEX_SHIFT(0) + }; + + template<> + const uint64_t bit_table::bit_table_[block_info::bits_per_block] = + { + CINDEX_SHIFT(56) + CINDEX_SHIFT(48) + CINDEX_SHIFT(40) + CINDEX_SHIFT(32) + CINDEX_SHIFT(24) + CINDEX_SHIFT(16) + CINDEX_SHIFT(8) + CINDEX_SHIFT(0) + }; + +#undef CINDEX_SHIFT + */ +} + diff --git a/fawnds/cindex/bit_access.hpp b/fawnds/cindex/bit_access.hpp new file mode 100644 index 0000000..a85b075 --- /dev/null +++ b/fawnds/cindex/bit_access.hpp @@ -0,0 +1,149 @@ +#pragma once + +#include "common.hpp" +#include "block_info.hpp" +#include +//#include // for debug + +namespace cindex +{ + // example internal representation for uint8_t blocks + // + // block index: 0 1 2 3 + // in-block offset: 76543210 76543210 76543210 76543210 + // +--------+--------+--------+--------+ + // blocks: |01101011|00011010|10111010|110 | + // +--------+--------+--------+--------+ + // bit_access index: 0 1 2 + // 01234567 89012345 67890123 456 + // + // rationale of filling MSB first (despite of complexity in calculating in-block offsets): + // the whole representation of blocks resembles a big-endian number (if the last block is completely filled), + // so appending or extracting a block of bits is more natural. + + /* + template + class bit_table + { + public: + static BlockType + get(std::size_t i) CINDEX_WARN_UNUSED_RESULT + { + return bit_table_[i]; + } + + private: + static const BlockType bit_table_[block_info::bits_per_block]; + }; + */ + + class bit_access + { + public: + template + static std::size_t + block_index(std::size_t i) + { + return i >> block_info::log_bits_per_block; + } + + template + static std::size_t + in_block_offset(std::size_t i) + { + return i & block_info::bit_mask; + } + + // single bit operations + template + static void + set(BlockType* v, std::size_t i) + { + v[block_index(i)] |= static_cast(BlockType(1) << (block_info::bits_per_block - 1 - in_block_offset(i))); + //v[block_index(i)] |= bit_table::get(in_block_offset(i)); + } + + template + static void + unset(BlockType* v, std::size_t i) + { + v[block_index(i)] &= static_cast(~(BlockType(1) << (block_info::bits_per_block - 1 - in_block_offset(i)))); + //v[block_index(i)] &= static_cast(~bit_table::get(in_block_offset(i))); + } + + template + static void + flip(BlockType* v, std::size_t i) + { + v[block_index(i)] ^= static_cast(BlockType(1) << (block_info::bits_per_block - 1 - in_block_offset(i))); + //v[block_index(i)] ^= bit_table::get(in_block_offset(i)); + } + + template + static bool + get(const BlockType* v, std::size_t i) //CINDEX_WARN_UNUSED_RESULT + { + return (v[block_index(i)] & (BlockType(1) << (block_info::bits_per_block - 1 - in_block_offset(i)))) != 0; + } + + // multiple bits operations + template + static void + copy_set(DestBlockType* dest, std::size_t dest_i, const SrcBlockType* src, std::size_t src_i, std::size_t count) + { + // get dest_off and src_off in each block's range + dest += block_index(dest_i); + src += block_index(src_i); + dest_i = in_block_offset(dest_i); + src_i = in_block_offset(src_i); + + while (count != 0) + { + // the remaining free space in the dest block + std::size_t dest_available = block_info::bits_per_block - dest_i; + // the remaining bits in the src block + std::size_t src_available = block_info::bits_per_block - src_i; + // the actual copiable bits + std::size_t copy_len = std::min(std::min(dest_available, src_available), count); + + // fetch a block from the source + SrcBlockType src_v = *src; + //std::cout << "1: " << src_v << std::endl; + // trim off the higher bits before the range being copied and align bits at MSB + src_v = static_cast(src_v << src_i); + //std::cout << "2: " << src_v << std::endl; + // trim off the lower bits after the range being copied and align bits at LSB + src_v = static_cast(src_v >> (block_info::bits_per_block - copy_len)); + //std::cout << "3: " << src_v << std::endl; + // copy the value to the dest-like block + DestBlockType dest_v = static_cast(src_v); + assert(static_cast(dest_v) == src_v); + //std::cout << "4: " << dest_v << std::endl; + // rearrange bits for the dest block + dest_v = static_cast(dest_v << (block_info::bits_per_block - dest_i - copy_len)); + //std::cout << "5: " << dest_v << std::endl; + // merge to the dest block (this does not clear out the existing bits at the dest) + *dest |= dest_v; + //std::cout << "6: " << dest << std::endl; + + // update the current block locations + if (dest_available == copy_len) + { + dest++; + dest_i = 0; + } + else + dest_i += copy_len; + if (src_available == copy_len) + { + src++; + src_i = 0; + } + else + src_i += copy_len; + count -= copy_len; + } + } + }; +} + diff --git a/fawnds/cindex/bit_vector.cpp b/fawnds/cindex/bit_vector.cpp new file mode 100644 index 0000000..168bfcf --- /dev/null +++ b/fawnds/cindex/bit_vector.cpp @@ -0,0 +1,71 @@ +#include "bit_vector.hpp" +#include + +namespace cindex +{ + template + bit_vector::bit_vector() + : buf_(NULL), size_(0), capacity_(8) + { + resize(); + } + + template + bit_vector::~bit_vector() + { + clear(); + } + + template + void + bit_vector::clear() + { + free(buf_); + buf_ = NULL; + + size_ = 0; + capacity_ = 0; + } + + template + void + bit_vector::compact() + { + capacity_ = size_; + resize(); + } + + template + void + bit_vector::resize() + { + std::size_t old_byte_size = block_info::size(size_); + std::size_t new_byte_size = block_info::size(capacity_); + + block_type* new_buf = reinterpret_cast(realloc(reinterpret_cast(buf_), new_byte_size)); + + if (!new_buf) + { + assert(buf_); + assert(new_byte_size > old_byte_size); + + new_buf = reinterpret_cast(malloc(new_byte_size)); + assert(new_buf); + + memcpy(new_buf, buf_, old_byte_size); + + free(buf_); + } + + buf_ = new_buf; + + if (new_byte_size > old_byte_size) + memset(reinterpret_cast(new_buf) + old_byte_size, 0, new_byte_size - old_byte_size); + } + + template class bit_vector; + template class bit_vector; + template class bit_vector; + template class bit_vector; +} + diff --git a/fawnds/cindex/bit_vector.hpp b/fawnds/cindex/bit_vector.hpp new file mode 100644 index 0000000..2f0032e --- /dev/null +++ b/fawnds/cindex/bit_vector.hpp @@ -0,0 +1,148 @@ +#pragma once + +#include "common.hpp" +#include "block_info.hpp" +#include "bit_access.hpp" + +namespace cindex +{ + template + class bit_vector + { + public: + typedef BlockType block_type; + + bit_vector(); + + ~bit_vector(); + + void clear(); + + std::size_t + size() const CINDEX_WARN_UNUSED_RESULT + { + return size_; + } + + // single bit operations + void + push_back(bool b) + { + if (size_ == capacity_) + { + if (capacity_ == 0) + capacity_ = 1; + else + capacity_ <<= 1; + resize(); + } + + if (b) + bit_access::set(buf_, size_); + size_++; + } + + void + pop_back() + { + assert(size_ > 0); + size_--; + bit_access::unset(buf_, size_); + } + + bool + operator[](std::size_t i) const CINDEX_WARN_UNUSED_RESULT + { + assert(i < size_); + return bit_access::get(buf_, i); + } + + bool + operator[](std::size_t i) CINDEX_WARN_UNUSED_RESULT + { + assert(i < size_); + return bit_access::get(buf_, i); + } + + // multiple bit operations + template + void + append(const bit_vector& r, std::size_t i, std::size_t len) + { + assert(i + len <= r.size()); + std::size_t target_size = size_ + r.size(); + if (target_size > capacity_) + { + if (capacity_ == 0) + capacity_ = 1; + while (target_size > capacity_) + capacity_ <<= 1; + resize(); + } + + bit_access::copy_set(buf_, size_, r.buf_, i, len); + size_ = target_size; + } + + template + void + append(const SrcBlockType* r, std::size_t i, std::size_t len) + { + std::size_t target_size = size_ + len; + if (target_size > capacity_) + { + if (capacity_ == 0) + capacity_ = 1; + while (target_size > capacity_) + capacity_ <<= 1; + resize(); + } + + bit_access::copy_set(buf_, size_, r, i, len); + size_ = target_size; + } + + template + void + append(T v, std::size_t len) //CINDEX_WARN_UNUSED_RESULT + { + std::size_t target_size = size_ + len; + if (target_size > capacity_) + { + if (capacity_ == 0) + capacity_ = 1; + while (target_size > capacity_) + capacity_ <<= 1; + resize(); + } + + bit_access::copy_set(buf_, size_, &v, block_info::bits_per_block - len, len); + size_ = target_size; + } + + template + T + get(std::size_t i, std::size_t len) const //CINDEX_WARN_UNUSED_RESULT + { + assert(len <= block_info::bits_per_block); + + T v = 0; + bit_access::copy_set(&v, block_info::bits_per_block - len, buf_, i, len); + return v; + } + + // TODO: append consecutive 0's or 1's for fast exp-golomb coding encoding + // TODO: count consecutive 0's or 1's starting at some position for faster exp-golomb coding decoding + + void compact(); + + protected: + void resize(); + + private: + block_type* buf_; + std::size_t size_; + std::size_t capacity_; + }; +} + diff --git a/fawnds/cindex/block_info.hpp b/fawnds/cindex/block_info.hpp new file mode 100644 index 0000000..d04a942 --- /dev/null +++ b/fawnds/cindex/block_info.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include "common.hpp" +#include + +namespace cindex +{ + template + class block_info + { + public: + typedef BlockType block_type; + + static const std::size_t bytes_per_block = sizeof(block_type); + static const std::size_t bits_per_block = bytes_per_block * 8; + static const std::size_t bit_mask = bits_per_block - 1; + static const std::size_t log_bits_per_block = boost::static_log2::value; + + static std::size_t + block_count(std::size_t n) CINDEX_WARN_UNUSED_RESULT + { + return (n + bits_per_block - 1) / bits_per_block; + } + + static std::size_t + size(std::size_t n) CINDEX_WARN_UNUSED_RESULT + { + return block_count(n) * bytes_per_block; + } + }; +} + diff --git a/fawnds/cindex/bucketing_index.cpp b/fawnds/cindex/bucketing_index.cpp new file mode 100644 index 0000000..6f0dc84 --- /dev/null +++ b/fawnds/cindex/bucketing_index.cpp @@ -0,0 +1,271 @@ +#include "bucketing_index.hpp" +#include + +// for template instantiation +#include "flat_absoff_bucketing.hpp" +#include "twolevel_absoff_bucketing.hpp" +#include "twolevel_reloff_bucketing.hpp" +#include "semi_direct_16_absoff_bucketing.hpp" +#include "semi_direct_16_reloff_bucketing.hpp" + +namespace cindex +{ + template + bucketing_index::bucketing_index(std::size_t key_len, std::size_t n, std::size_t bucket_size, std::size_t dest_base, std::size_t dest_keys_per_block, std::size_t skip_bits) + : key_len_(key_len), n_(n), dest_base_(dest_base), dest_keys_per_block_(dest_keys_per_block), skip_bits_(skip_bits) + { + bucket_bits_ = 0; + while ((guarded_cast(1) << bucket_bits_) < (n_ / bucket_size)) + bucket_bits_++; + bucket_count_ = guarded_cast(1) << bucket_bits_; + bucket_size_ = n_ / bucket_count_; + + bucketing_.resize(bucket_count_ + 1, bucket_size_, dest_keys_per_block_); + bucketing_.insert(0, 0); + + last_dest_offset_ = 0; + pending_bucket_ = 0; + /* + pending_keys_ = new key_array_type(key_len, bucket_size * 2 + 256); + assert(pending_keys_); + */ + pending_key_count_ = 0; + } + + template + bucketing_index::bucketing_index(int fd, off_t offset) + { + ssize_t len = load_from_file(fd, offset); + (void)len; + } + + template + bucketing_index::~bucketing_index() + { + if (!finalized()) + flush(); + } + + template + bool + bucketing_index::finalized() const + { + return pending_bucket_ == bucket_count_; + } + + template + bool + bucketing_index::insert(const uint8_t* key) + { + assert(!finalized()); + + std::size_t bucket = find_bucket(key); + + // key order and duplicity checking + //assert(bucket >= pending_bucket_); + //assert(pending_key_count_ == 0 || memcmp((*pending_keys_)[pending_key_count_ - 1], key, key_len_) < 0); + + if (pending_bucket_ > bucket || + (pending_key_count_ != 0 && memcmp(pending_keys_[pending_key_count_ - 1], key, key_len_) >= 0)) + { + std::cerr << "cannot insert a non-sorted key" << std::endl; + return false; + } + + while (pending_bucket_ < bucket) + index_pending_keys(); + + //assert(pending_key_count_ < pending_keys_->size()); + //memcpy((*pending_keys_)[pending_key_count_++], key, key_len_); + uint8_t* key_c = new uint8_t[key_len_]; + memcpy(key_c, key, key_len_); + pending_keys_.push_back(key_c); + pending_key_count_++; + return true; + } + + template + void + bucketing_index::flush() + { + assert(!finalized()); + + while (pending_bucket_ < bucket_count_) + index_pending_keys(); + + /* + delete pending_keys_; + pending_keys_ = NULL; + */ + + if (finalized()) + bucketing_.finalize(); + } + + struct bucketing_index_state + { + std::size_t key_len_; + std::size_t n_; + + std::size_t bucket_size_; + + std::size_t dest_base_; + std::size_t dest_keys_per_block_; + std::size_t skip_bits_; + + // could be calculated but included for simplicity + std::size_t bucket_count_; + std::size_t bucket_bits_; + }; + + template + ssize_t bucketing_index::load_from_file(int fd, off_t offset) + { + bucketing_index_state state; + ssize_t read_len = pread(fd, &state, sizeof(state), offset); + assert(read_len == sizeof(state)); + + key_len_ = state.key_len_; + n_ = state.n_; + bucket_size_ = state.bucket_size_; + dest_base_ = state.dest_base_; + dest_keys_per_block_ = state.dest_keys_per_block_; + skip_bits_ = state.skip_bits_; + bucket_count_ = state.bucket_count_; + bucket_bits_ = state.bucket_bits_; + + // TODO: read bucketing information + // TODO: read trie information + return 0; + } + + template + ssize_t bucketing_index::store_to_file(int fd, off_t offset) + { + assert(finalized()); + + bucketing_index_state state; + + state.key_len_ = key_len_; + state.n_ = n_; + state.bucket_size_ = bucket_size_; + state.dest_base_ = dest_base_; + state.dest_keys_per_block_ = dest_keys_per_block_; + state.skip_bits_ = skip_bits_; + state.bucket_count_ = bucket_count_; + state.bucket_bits_ = bucket_bits_; + + ssize_t wrote_len = pwrite(fd, &state, sizeof(state), offset); + assert(wrote_len == sizeof(state)); + + // TODO: write bucketing information + // TODO: write trie information + return 0; + } + + template + std::size_t + bucketing_index::locate(const uint8_t* key) const + { + assert(finalized()); + + std::size_t bucket = find_bucket(key); + + std::size_t iter = bucketing_.index_offset(bucket); // no alignment + //std::size_t iter = bucketing_.index_offset(bucket) * 4; // 4-bit aligned + std::size_t off = bucketing_.dest_offset(bucket); + std::size_t n = bucketing_.dest_offset(bucket + 1) - off; + + std::size_t key_index = off + trie_.locate( + repr_, iter, + key, key_len_, + 0, n, + dest_base_ + off, dest_keys_per_block_, + skip_bits_ + bucket_bits_ + ); + + //printf("key_index = %lu\n", key_index); + return key_index; + } + + template + std::size_t + bucketing_index::find_bucket(const uint8_t* key) const + { + std::size_t bits = 0; + std::size_t index = 0; + while (bits < bucket_bits_) + { + index <<= 1; + if (skip_bits_ + bits > key_len_ * 8) + { + // too short key + assert(false); + break; + } + if (bit_access::get(key, skip_bits_ + bits)) + index |= 1; + bits++; + } + + return index; + } + + template + void + bucketing_index::index_pending_keys() + { + assert(pending_bucket_ < bucket_count_); + + trie_.encode( + repr_, + //*pending_keys_, key_len_, + pending_keys_, key_len_, + 0, pending_key_count_, + dest_base_ + last_dest_offset_, dest_keys_per_block_, + skip_bits_ + bucket_bits_ + ); + + pending_bucket_ += 1; + + // no alignment + bucketing_.insert(repr_.size(), last_dest_offset_ + pending_key_count_); + + // add padding for 4-bit alignment + //std::size_t padding = (4 - (repr_.size() & 3)) & 3; + //while (padding--) + // repr_.push_back(0); + //assert((repr_.size() & 3) == 0); + //bucketing_.insert(repr_.size() / 4, last_dest_offset_ + pending_key_count_); + + last_dest_offset_ += pending_key_count_; + + for (std::size_t i = 0; i < pending_key_count_; i++) + delete [] pending_keys_[i]; + pending_keys_.clear(); + pending_key_count_ = 0; + } + + template + std::size_t + bucketing_index::bit_size_trie_only() const + { + assert(finalized()); + return repr_.size(); + } + + template + std::size_t + bucketing_index::bit_size() const + { + assert(finalized()); + return bit_size_trie_only() + bucketing_.bit_size(); + } + + template class bucketing_index >; + template class bucketing_index >; + template class bucketing_index; + template class bucketing_index; + template class bucketing_index; +} + diff --git a/fawnds/cindex/bucketing_index.hpp b/fawnds/cindex/bucketing_index.hpp new file mode 100644 index 0000000..b12e4ef --- /dev/null +++ b/fawnds/cindex/bucketing_index.hpp @@ -0,0 +1,68 @@ +#pragma once + +#include "common.hpp" +#include "bit_vector.hpp" +#include "key_array.hpp" +#include "trie.hpp" +#include + +namespace cindex +{ + template + class bucketing_index + { + public: + typedef BucketingType bucketing_type; + + public: + bucketing_index(std::size_t key_len, std::size_t n, std::size_t bucket_size, std::size_t dest_base = 0, std::size_t dest_keys_per_block = 1, std::size_t skip_bits = 0); + bucketing_index(int fd, off_t offset); + ~bucketing_index(); + + bool finalized() const CINDEX_WARN_UNUSED_RESULT; + + bool insert(const uint8_t* key); + void flush(); + + ssize_t store_to_file(int fd, off_t offset) CINDEX_WARN_UNUSED_RESULT; + + std::size_t locate(const uint8_t* key) const CINDEX_WARN_UNUSED_RESULT; + + std::size_t bit_size_trie_only() const CINDEX_WARN_UNUSED_RESULT; + std::size_t bit_size() const CINDEX_WARN_UNUSED_RESULT; + + protected: + std::size_t find_bucket(const uint8_t* key) const CINDEX_WARN_UNUSED_RESULT; + void index_pending_keys(); + + ssize_t load_from_file(int fd, off_t offset) CINDEX_WARN_UNUSED_RESULT; + + typedef bit_vector<> vector_type; + typedef key_array key_array_type; + + private: + std::size_t key_len_; + std::size_t n_; + + std::size_t bucket_size_; + + std::size_t dest_base_; + std::size_t dest_keys_per_block_; + std::size_t skip_bits_; + + std::size_t bucket_count_; + std::size_t bucket_bits_; + + vector_type repr_; + trie<> trie_; + + bucketing_type bucketing_; + + std::size_t last_dest_offset_; + std::size_t pending_bucket_; + //key_array_type* pending_keys_; + std::vector pending_keys_; + std::size_t pending_key_count_; + }; +} + diff --git a/fawnds/cindex/common.hpp b/fawnds/cindex/common.hpp new file mode 100644 index 0000000..6d88ccb --- /dev/null +++ b/fawnds/cindex/common.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include +#include + +#define CINDEX_UNUSED __attribute__((unused)) +#define CINDEX_WARN_UNUSED_RESULT __attribute__((warn_unused_result)) + +#define CINDEX_LITTLE_ENDIAN + +#include "basic_types.hpp" + diff --git a/fawnds/cindex/exp_golomb.hpp b/fawnds/cindex/exp_golomb.hpp new file mode 100644 index 0000000..247d903 --- /dev/null +++ b/fawnds/cindex/exp_golomb.hpp @@ -0,0 +1,96 @@ +#pragma once + +#include "common.hpp" +#include "bit_access.hpp" + +namespace cindex +{ + template + class exp_golomb + { + public: + template + static void encode(BufferType& out_buf, const T& n) + { + BOOST_STATIC_ASSERT(sizeof(T) * 8 >= Order); + + T m; + if (Order) + m = (n >> Order) + 1; + else + m = n + 1; + + int len = 0; + { + T p = m; + while (p) + { + len++; + p >>= 1; + } + } + + for (int i = 1; i < len; i++) + out_buf.push_back(0); + + assert(m >> (len - 1) == 1); + out_buf.push_back(1); + + //for (int i = len - 2; i >= 0; i--) + // out_buf.push_back((m >> i) & 1); + out_buf.append(m, static_cast(len - 1)); + + if (Order) + { + //for (int i = guarded_cast(Order) - 1; i >= 0; i--) + // out_buf.push_back((n >> i) & 1); + out_buf.append(n, static_cast(Order)); + } + } + + template + static T decode(const BufferType& in_buf, std::size_t& in_out_buf_iter) + { + BOOST_STATIC_ASSERT(sizeof(T) * 8 >= Order); + + int len = 1; + while (true) + { + assert(in_out_buf_iter < in_buf.size()); + if (in_buf[in_out_buf_iter++]) + break; + len++; + } + + T m = guarded_cast(1) << (len - 1); + //for (int i = len - 2; i >= 0; i--) + //{ + // assert(in_out_buf_iter < in_buf.size()); + // if (in_buf[in_out_buf_iter++]) + // m |= guarded_cast(1) << i; + //} + assert(in_out_buf_iter + static_cast(len - 1) <= in_buf.size()); + m |= in_buf.template get(in_out_buf_iter, static_cast(len - 1)); // "template" prefix is used to inform the compiler that in_buf.get is a member template + in_out_buf_iter += static_cast(len - 1); + + T n; + if (Order) + { + n = (m - 1) << Order; + assert(in_out_buf_iter + Order <= in_buf.size()); + //for (int i = guarded_cast(Order) - 1; i >= 0; i--) + //{ + // if (in_buf[in_out_buf_iter++]) + // n |= guarded_cast(1) << i; + //} + n |= in_buf.template get(in_out_buf_iter, static_cast(Order)); + in_out_buf_iter += static_cast(Order); + } + else + n = m - 1; + + return n; + } + }; +} + diff --git a/fawnds/cindex/expected_size.cpp b/fawnds/cindex/expected_size.cpp new file mode 100644 index 0000000..3b44998 --- /dev/null +++ b/fawnds/cindex/expected_size.cpp @@ -0,0 +1,102 @@ +#include "expected_size.hpp" +#include + +namespace cindex +{ + static struct + { + std::size_t keys_per_bucket; + std::size_t keys_per_block; + bool weak_ranking; + double bits_per_key; + } known_index_sizes[] = { + {1, 1, false, 0.000000}, + {2, 1, false, 1.500000}, + {4, 1, false, 2.035714}, + {8, 1, false, 2.368062}, + {16, 1, false, 2.565474}, + {32, 1, false, 2.719423}, + {64, 1, false, 2.800122}, + {128, 1, false, 2.846470}, + {256, 1, false, 2.872846}, + {512, 1, false, 2.887720}, + {1024, 1, false, 2.896029}, + + {1, 1, true, 0.000000}, + {2, 1, true, 1.000000}, + {4, 1, true, 1.595238}, + {8, 1, true, 1.925228}, + {16, 1, true, 2.122761}, + {32, 1, true, 2.276723}, + {64, 1, true, 2.357425}, + {128, 1, true, 2.403773}, + {256, 1, true, 2.430150}, + {512, 1, true, 2.445024}, + {1024, 1, true, 2.453333}, + + {1, 4, false, 0.000000}, + {2, 4, false, 0.375000}, + {4, 4, false, 0.750000}, + {8, 4, false, 1.153312}, + {16, 4, false, 1.355466}, + {32, 4, false, 1.509626}, + {64, 4, false, 1.590345}, + {128, 4, false, 1.636697}, + {256, 4, false, 1.663074}, + {512, 4, false, 1.677949}, + {1024, 4, false, 1.686258}, + + {1, 4, true, 0.000000}, + {2, 4, true, 0.250000}, + {4, 4, true, 0.651786}, + {8, 4, true, 1.048069}, + {16, 4, true, 1.249690}, + {32, 4, true, 1.403818}, + {64, 4, true, 1.484533}, + {128, 4, true, 1.530884}, + {256, 4, true, 1.557261}, + {512, 4, true, 1.572136}, + {1024, 4, true, 1.580445}, + + {1, 16, false, 0.000000}, + {2, 16, false, 0.093750}, + {4, 16, false, 0.130371}, + {8, 16, false, 0.202756}, + {16, 16, false, 0.382474}, + {32, 16, false, 0.530633}, + {64, 16, false, 0.609876}, + {128, 16, false, 0.655798}, + {256, 16, false, 0.682011}, + {512, 16, false, 0.696813}, + {1024, 16, false, 0.705088}, + + {1, 16, true, 0.000000}, + {2, 16, true, 0.062500}, + {4, 16, true, 0.119838}, + {8, 16, true, 0.197576}, + {16, 16, true, 0.377405}, + {32, 16, true, 0.525653}, + {64, 16, true, 0.604917}, + {128, 16, true, 0.650845}, + {256, 16, true, 0.677060}, + {512, 16, true, 0.691863}, + {1024, 16, true, 0.700139}, + }; + + double + expected_size::index_size(std::size_t keys_per_bucket CINDEX_UNUSED, std::size_t keys_per_block CINDEX_UNUSED) + { + const bool weak_ranking = false; + for (std::size_t i = 0; i < sizeof(known_index_sizes) / sizeof(known_index_sizes[0]); i++) + { + if (known_index_sizes[i].keys_per_bucket == keys_per_bucket && + known_index_sizes[i].keys_per_block == keys_per_block && + known_index_sizes[i].weak_ranking == weak_ranking) + return known_index_sizes[i].bits_per_key; + } + std::cerr << "bug: no index size information availble for keys_per_bucket = " << keys_per_bucket << ", keys_per_block = " << keys_per_block << std::endl; + //assert(false); + return 3.; + } +} + diff --git a/fawnds/cindex/expected_size.hpp b/fawnds/cindex/expected_size.hpp new file mode 100644 index 0000000..e538245 --- /dev/null +++ b/fawnds/cindex/expected_size.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "common.hpp" + +namespace cindex +{ + class expected_size + { + public: + static double index_size(std::size_t keys_per_bucket, std::size_t keys_per_block); + }; +} + diff --git a/fawnds/cindex/flat_absoff_bucketing.cpp b/fawnds/cindex/flat_absoff_bucketing.cpp new file mode 100644 index 0000000..e7d6ae6 --- /dev/null +++ b/fawnds/cindex/flat_absoff_bucketing.cpp @@ -0,0 +1,72 @@ +#include "flat_absoff_bucketing.hpp" +#include + +namespace cindex +{ + template + flat_absoff_bucketing::flat_absoff_bucketing(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block CINDEX_UNUSED) + { + resize(size, keys_per_bucket, keys_per_block); + } + + template + void + flat_absoff_bucketing::resize(std::size_t size, std::size_t keys_per_bucket CINDEX_UNUSED, std::size_t keys_per_block CINDEX_UNUSED) + { + size_ = size; + + bucket_info_.resize(size); + current_i_ = 0; + + //if (size_) + // std::cout << "bucket_count: " << size << std::endl; + } + + template + void + flat_absoff_bucketing::insert(const std::size_t& index_offset, const std::size_t& dest_offset) + { + assert(current_i_ < size_); + + bucket_info_[current_i_][0] = guarded_cast(index_offset); + bucket_info_[current_i_][1] = guarded_cast(dest_offset); + + assert(this->index_offset(current_i_) == index_offset); + assert(this->dest_offset(current_i_) == dest_offset); + + current_i_++; + } + + template + std::size_t + flat_absoff_bucketing::index_offset(std::size_t i) const + { + assert(i < size_); + return bucket_info_[i][0]; + } + + template + std::size_t + flat_absoff_bucketing::dest_offset(std::size_t i) const + { + assert(i < size_); + return bucket_info_[i][1]; + } + + template + std::size_t + flat_absoff_bucketing::size() const + { + return size_; + } + + template + std::size_t + flat_absoff_bucketing::bit_size() const + { + return bucket_info_.size() * 2 * sizeof(value_type) * 8; + } + + template class flat_absoff_bucketing<>; +} + diff --git a/fawnds/cindex/flat_absoff_bucketing.hpp b/fawnds/cindex/flat_absoff_bucketing.hpp new file mode 100644 index 0000000..ad018a1 --- /dev/null +++ b/fawnds/cindex/flat_absoff_bucketing.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include "common.hpp" +#include +#include + +namespace cindex +{ + template + class flat_absoff_bucketing + { + public: + typedef ValueType value_type; + + public: + flat_absoff_bucketing(std::size_t size = 0, std::size_t keys_per_bucket = 1, std::size_t keys_per_block = 1); + + void resize(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block); + + void insert(const std::size_t& index_offset, const std::size_t& dest_offset); + void finalize() {} + + std::size_t index_offset(std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + std::size_t dest_offset(std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + + std::size_t size() const CINDEX_WARN_UNUSED_RESULT; + + std::size_t bit_size() const CINDEX_WARN_UNUSED_RESULT; + + private: + std::size_t size_; + + std::vector > bucket_info_; + std::size_t current_i_; + }; +} + diff --git a/fawnds/cindex/huffman.hpp b/fawnds/cindex/huffman.hpp new file mode 100644 index 0000000..a1da7f3 --- /dev/null +++ b/fawnds/cindex/huffman.hpp @@ -0,0 +1,321 @@ +#pragma once + +#include "common.hpp" +#include "bit_access.hpp" +#include +#include + +namespace cindex +{ + template + class huffman_tree + { + public: + typedef RefType ref_type; + + public: + huffman_tree(std::size_t num_symbols) + : num_symbols_(num_symbols) + { + assert(guarded_cast(guarded_cast(num_symbols_ * 2 - 1)) + == num_symbols_ * 2 - 1); + nodes_ = new ref_type[(num_symbols_ - 1) * 2]; + } + + huffman_tree(const huffman_tree& t) + : num_symbols_(t.num_symbols_) + { + nodes_ = new ref_type[(num_symbols_ - 1) * 2]; + memcpy(nodes_, t.nodes_, sizeof(ref_type) * (num_symbols_ - 1) * 2); + } + + ~huffman_tree() { delete [] nodes_; } + + std::size_t num_symbols() const CINDEX_WARN_UNUSED_RESULT { return num_symbols_; } + + bool is_symbol(ref_type ref) const CINDEX_WARN_UNUSED_RESULT { return ref < num_symbols_; } + std::size_t symbol(ref_type ref) const CINDEX_WARN_UNUSED_RESULT { return guarded_cast(ref); } + + ref_type root() CINDEX_WARN_UNUSED_RESULT { return guarded_cast(num_symbols_ * 2 - 2); } + const ref_type root() const CINDEX_WARN_UNUSED_RESULT { return guarded_cast(num_symbols_ * 2 - 2); } + + ref_type& left(ref_type ref) CINDEX_WARN_UNUSED_RESULT + { + assert(!is_symbol(ref) && ref < num_symbols_ * 2 - 1); + return nodes_[(ref - guarded_cast(num_symbols_)) * 2]; + } + const ref_type& left(ref_type ref) const CINDEX_WARN_UNUSED_RESULT + { + assert(!is_symbol(ref) && ref < num_symbols_ * 2 - 1); + return nodes_[(ref - guarded_cast(num_symbols_)) * 2]; + } + + ref_type& right(ref_type ref) CINDEX_WARN_UNUSED_RESULT + { + assert(!is_symbol(ref) && ref < num_symbols_ * 2 - 1); + return nodes_[(ref - guarded_cast(num_symbols_)) * 2 + 1]; + } + const ref_type& right(ref_type ref) const CINDEX_WARN_UNUSED_RESULT + { + assert(!is_symbol(ref) && ref < num_symbols_ * 2 - 1); + return nodes_[(ref - guarded_cast(num_symbols_)) * 2 + 1]; + } + + private: + std::size_t num_symbols_; + ref_type* nodes_; + }; + + //! huffman tree generator + template + class huffman_tree_generator + { + public: + typedef FreqType freq_type; + + public: + huffman_tree_generator(std::size_t num_symbols) + : num_symbols_(num_symbols) + { + freqs_ = new FreqType[num_symbols_ * 2 - 1]; + } + + ~huffman_tree_generator() { delete [] freqs_; } + + std::size_t num_symbols() const CINDEX_WARN_UNUSED_RESULT { return num_symbols_; } + + FreqType& operator[](std::size_t symbol) CINDEX_WARN_UNUSED_RESULT { assert(symbol < num_symbols_); return freqs_[symbol]; } + + const FreqType& operator[](std::size_t symbol) const CINDEX_WARN_UNUSED_RESULT { assert(symbol < num_symbols_); return freqs_[symbol]; } + + template + void generate(huffman_tree& out_t) const + { + assert(out_t.num_symbols() == num_symbols_); + + std::multimap q; + RefType ref = 0; + for (; ref < num_symbols_; ref++) + q.insert(std::make_pair(freqs_[ref], ref)); + + for (; ref < num_symbols_ * 2 - 1; ref++) + { + RefType left = (*q.begin()).second; + q.erase(q.begin()); + RefType right = (*q.begin()).second; + q.erase(q.begin()); + + FreqType freq = freqs_[ref] = freqs_[left] + freqs_[right]; + out_t.left(ref) = left; + out_t.right(ref) = right; + + q.insert(std::make_pair(freq, ref)); + } + } + + private: + std::size_t num_symbols_; + FreqType* freqs_; + }; + + class huffman_table + { + public: + typedef uint32_t block_type; + typedef uint8_t length_type; + + public: + template + huffman_table(const huffman_tree& t) + { + num_symbols_ = t.num_symbols(); + lengths_ = new length_type[num_symbols_]; + codes_ = new block_type*[num_symbols_]; + for (std::size_t symbol = 0; symbol < num_symbols_; symbol++) + codes_[symbol] = NULL; + + block_type prefix[block_info::block_count(num_symbols_)]; + generate(t, t.root(), prefix, 0); + } + + ~huffman_table() + { + for (std::size_t symbol = 0; symbol < num_symbols_; symbol++) + if (codes_[symbol]) + delete [] codes_[symbol]; + delete [] codes_; + delete [] lengths_; + } + + std::size_t num_symbols() const CINDEX_WARN_UNUSED_RESULT { return num_symbols_; } + + std::size_t length(std::size_t symbol) const CINDEX_WARN_UNUSED_RESULT { assert(symbol < num_symbols_); return lengths_[symbol]; } + const block_type* code(std::size_t symbol) const CINDEX_WARN_UNUSED_RESULT { assert(symbol < num_symbols_); return codes_[symbol]; } + + protected: + template + void generate(const huffman_tree& t, RefType p, block_type* prefix, std::size_t depth) + { + if (t.is_symbol(p)) + { + std::size_t symbol = t.symbol(p); + assert(depth < (1 << block_info::bits_per_block)); + lengths_[symbol] = static_cast(depth); + codes_[symbol] = new block_type[block_info::block_count(depth)]; + memcpy(codes_[symbol], prefix, block_info::size(depth)); + } + else + { + bit_access::unset(prefix, depth); + generate(t, t.left(p), prefix, depth + 1); + bit_access::set(prefix, depth); + generate(t, t.right(p), prefix, depth + 1); + } + } + + private: + std::size_t num_symbols_; + length_type* lengths_; + block_type** codes_; + }; + + template + class huffman_rev_table + { + public: + typedef RefType ref_type; + typedef uint8_t length_type; + + public: + huffman_rev_table(const huffman_tree& t) + : max_length_(0) + { + assert(t.num_symbols() <= (1 << block_info::bits_per_block)); + find_max_length(t, t.root(), 0); + assert(max_length_ <= (1 << block_info::bits_per_block)); + + refs_ = new ref_type[1 << max_length_]; + lengths_ = new length_type[1 << max_length_]; + fill_table(t, t.root(), 0, 0); + } + + ~huffman_rev_table() + { + delete [] refs_; + delete [] lengths_; + } + + std::size_t max_length() const CINDEX_WARN_UNUSED_RESULT { return max_length_; } + + ref_type symbol(std::size_t v) const CINDEX_WARN_UNUSED_RESULT { assert(v < (1u << max_length_)); return refs_[v]; } + std::size_t length(std::size_t v) const CINDEX_WARN_UNUSED_RESULT { assert(v < (1u << max_length_)); return lengths_[v]; } + + protected: + void find_max_length(const huffman_tree& t, ref_type p, std::size_t depth) + { + if (t.is_symbol(p)) + { + if (max_length_ < depth) + max_length_ = depth; + } + else + { + find_max_length(t, t.left(p), depth + 1); + find_max_length(t, t.right(p), depth + 1); + } + } + + void fill_table(const huffman_tree& t, ref_type p, std::size_t numeric_prefix, std::size_t depth) + { + if (t.is_symbol(p)) + { + std::size_t padding = max_length_ - depth; + std::size_t base = numeric_prefix << padding; + for (std::size_t k = 0; k < (1u << padding); k++) + { + refs_[base + k] = p; + assert(depth < (1 << block_info::bits_per_block)); + lengths_[base + k] = static_cast(depth); + } + } + else + { + fill_table(t, t.left(p), (numeric_prefix << 1), depth + 1); + fill_table(t, t.right(p), (numeric_prefix << 1) | 1, depth + 1); + } + } + + private: + std::size_t max_length_; + ref_type* refs_; + length_type* lengths_; + }; + + template + class huffman + { + public: + typedef RefType ref_type; + typedef huffman_table::block_type block_type; + + public: + static const std::size_t nsymbol = std::size_t(-1); + + public: + huffman(const huffman_tree& t) : t_(t), tbl_(t), rev_tbl_(t) {} + + template + void encode(BufferType& out_buf, std::size_t symbol) const + { + std::size_t len = tbl_.length(symbol); + const block_type* code = tbl_.code(symbol); + + //for (std::size_t i = 0; i < len; i++) + // out_buf.push_back(bit_access::get(code, i)); + out_buf.append(code, 0, len); + } + + template + std::size_t decode(const BufferType& in_buf, std::size_t& in_out_buf_iter) const + { + if (in_out_buf_iter == in_buf.size()) + return nsymbol; + + // simple decoding is ~10% faster than table-based decoding for small codewords + ref_type p = t_.root(); + while (!t_.is_symbol(p)) + { + assert(in_out_buf_iter < in_buf.size()); + + if (!in_buf[in_out_buf_iter++]) + p = t_.left(p); + else + p = t_.right(p); + } + return t_.symbol(p); + + /* + std::size_t fetch_length = rev_tbl_.max_length(); + std::size_t padding = 0; + + if (fetch_length > in_buf.size() - in_out_buf_iter) + { + padding = fetch_length - (in_buf.size() - in_out_buf_iter); + fetch_length -= padding; + } + + std::size_t idx = in_buf.template get(in_out_buf_iter, fetch_length); + idx <<= padding; + + in_out_buf_iter += rev_tbl_.length(idx); + return rev_tbl_.symbol(idx); + */ + } + + private: + const huffman_tree t_; + const huffman_table tbl_; + const huffman_rev_table rev_tbl_; + }; +} + diff --git a/fawnds/cindex/key_array.cpp b/fawnds/cindex/key_array.cpp new file mode 100644 index 0000000..5c18d32 --- /dev/null +++ b/fawnds/cindex/key_array.cpp @@ -0,0 +1,54 @@ +#include "key_array.hpp" +#include + +namespace cindex +{ + key_array::key_array(std::size_t key_len, std::size_t size) + : key_len_(key_len), size_(size), v_(new uint8_t[key_len * size]) + { + assert(v_); + } + + key_array::~key_array() + { + delete [] v_; + v_ = NULL; + } + + const uint8_t* + key_array::operator[](std::size_t i) const + { + assert(i < size_); + return v_ + i * key_len_; + } + + uint8_t* + key_array::operator[](std::size_t i) + { + assert(i < size_); + return v_ + i * key_len_; + } + + std::size_t + key_array::size() const + { + return size_; + } + + void + key_array::generate_random_keys(std::size_t off, std::size_t n, unsigned int seed) + { + assert(off + n <= size_); + + uint8_t key[key_len_]; + + srand(seed); + for (std::size_t i = off; i < off + n; i++) + { + for (std::size_t j = 0; j < key_len_; j++) + key[j] = static_cast(rand()); + memcpy((*this)[i], key, key_len_); + } + } +} + diff --git a/fawnds/cindex/key_array.hpp b/fawnds/cindex/key_array.hpp new file mode 100644 index 0000000..9847b0d --- /dev/null +++ b/fawnds/cindex/key_array.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include "common.hpp" + +namespace cindex +{ + class key_array + { + public: + key_array(std::size_t key_len, std::size_t size); + + ~key_array(); + + const uint8_t* operator[](std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + + uint8_t* operator[](std::size_t i) CINDEX_WARN_UNUSED_RESULT; + + std::size_t key_len() const CINDEX_WARN_UNUSED_RESULT { return key_len_; } + std::size_t size() const CINDEX_WARN_UNUSED_RESULT; + + void generate_random_keys(std::size_t off, std::size_t n, unsigned int seed = 0); + + private: + std::size_t key_len_; + std::size_t size_; + uint8_t* v_; + }; +} + diff --git a/fawnds/cindex/semi_direct_16_absoff_bucketing.cpp b/fawnds/cindex/semi_direct_16_absoff_bucketing.cpp new file mode 100644 index 0000000..7a70d22 --- /dev/null +++ b/fawnds/cindex/semi_direct_16_absoff_bucketing.cpp @@ -0,0 +1,160 @@ +#include "semi_direct_16_absoff_bucketing.hpp" +#include + +namespace cindex +{ + const std::size_t semi_direct_16_absoff_bucketing::ranks_[16] = { + 0 /* unused */, 0, 1, 2, + 0 /* unused */, 3, 4, 5, + 0 /* unused */, 6, 7, 8, + 0 /* unused */, 9, 10, 11 + }; + + semi_direct_16_absoff_bucketing::semi_direct_16_absoff_bucketing(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block) + { + resize(size, keys_per_bucket, keys_per_block); + } + + void + semi_direct_16_absoff_bucketing::resize(std::size_t size, std::size_t keys_per_bucket CINDEX_UNUSED, std::size_t keys_per_block CINDEX_UNUSED) + { + if (size % 16 != 0) + size = (size + 15) / 16 * 16; + size_ = size; + + boost::array v; + v[0] = v[1] = 0u; + bucket_info_.resize(0); + bucket_info_.resize(size / 16 * 5, v); + current_i_ = 0; + + //if (size_) + // std::cout << "bucket_count: " << size << std::endl; + if (keys_per_bucket > 128) + std::cout << "warning: too high keys per bucket" << std::endl; + } + + void + semi_direct_16_absoff_bucketing::store(const std::size_t& idx, const std::size_t& type, const std::size_t& mask, const std::size_t& shift, const std::size_t& v) + { + assert((v & mask) == v); // overflow test + (void)mask; + bucket_info_[idx][type] |= guarded_cast(v << shift); + } + + std::size_t + semi_direct_16_absoff_bucketing::load(const std::size_t& idx, const std::size_t& type, const std::size_t& mask, const std::size_t& shift) const + { + return (bucket_info_[idx][type] >> shift) & mask; + } + + void + semi_direct_16_absoff_bucketing::insert(const std::size_t& index_offset, const std::size_t& dest_offset) + { + assert(current_i_ < size_); + + //std::cout << "i: " << current_i_ << std::endl; + //std::cout << "current: " << index_offset << " " << dest_offset << std::endl; + + std::size_t base_idx = current_i_ / 16 * 5; + if (current_i_ % 16 == 0) + { + store(base_idx + 0, 0, 0xffffffff, 0, index_offset); + store(base_idx + 0, 1, 0xffffffff, 0, dest_offset); + } + else if (current_i_ % 4 == 0) + { + std::size_t diff_base = current_i_ - 4; + std::size_t index_offset_delta = index_offset - last_index_offsets_[diff_base % 16]; + std::size_t dest_offset_delta = dest_offset - last_dest_offsets_[diff_base % 16]; + + store(base_idx + 1, 0, 0x3ff, ((current_i_ % 16) / 4 - 1) * 10, + index_offset_delta); + store(base_idx + 1, 1, 0x3ff, ((current_i_ % 16) / 4 - 1) * 10, + dest_offset_delta); + } + else + { + std::size_t diff_base = current_i_ - 1; + std::size_t index_offset_delta = index_offset - last_index_offsets_[diff_base % 16]; + std::size_t dest_offset_delta = dest_offset - last_dest_offsets_[diff_base % 16]; + std::size_t rank = ranks_[current_i_ % 16]; + + store(base_idx + 2 + rank / 4, 0, 0xff, rank % 4 * 8, + index_offset_delta); + store(base_idx + 2 + rank / 4, 1, 0xff, rank % 4 * 8, + dest_offset_delta); + } + + assert(this->index_offset(current_i_) == index_offset); + assert(this->dest_offset(current_i_) == dest_offset); + + last_index_offsets_[current_i_ % 16] = index_offset; + last_dest_offsets_[current_i_ % 16] = dest_offset; + + current_i_++; + } + + std::size_t + semi_direct_16_absoff_bucketing::index_offset(std::size_t i) const + { + assert(i < size_); + + std::size_t base_idx = i / 16 * 5; + std::size_t v; + if (i % 16 == 0) + v = load(base_idx + 0, 0, 0xffffffff, 0); + else if (i % 4 == 0) + { + std::size_t diff_base = i - 4; + v = load(base_idx + 1, 0, 0x3ff, ((i % 16) / 4 - 1) * 10) + index_offset(diff_base); + } + else + { + std::size_t diff_base = i - 1; + std::size_t rank = ranks_[i % 16]; + v = load(base_idx + 2 + rank / 4, 0, 0xff, rank % 4 * 8) + index_offset(diff_base); + } + + //std::cout << "index_offset(" << i << ") = " << v << std::endl; + return v; + } + + std::size_t + semi_direct_16_absoff_bucketing::dest_offset(std::size_t i) const + { + assert(i < size_); + + std::size_t base_idx = i / 16 * 5; + std::size_t v; + if (i % 16 == 0) + v = load(base_idx + 0, 1, 0xffffffff, 0); + else if (i % 4 == 0) + { + std::size_t diff_base = i - 4; + v = load(base_idx + 1, 1, 0x3ff, ((i % 16) / 4 - 1) * 10) + dest_offset(diff_base); + } + else + { + std::size_t diff_base = i - 1; + std::size_t rank = ranks_[i % 16]; + v = load(base_idx + 2 + rank / 4, 1, 0xff, rank % 4 * 8) + dest_offset(diff_base); + } + + //std::cout << "dest_offset(" << i << ") = " << v << std::endl; + return v; + } + + std::size_t + semi_direct_16_absoff_bucketing::size() const + { + return size_; + } + + std::size_t + semi_direct_16_absoff_bucketing::bit_size() const + { + return bucket_info_.size() * 2 * sizeof(uint32_t) * 8; + } +} + diff --git a/fawnds/cindex/semi_direct_16_absoff_bucketing.hpp b/fawnds/cindex/semi_direct_16_absoff_bucketing.hpp new file mode 100644 index 0000000..4e45d46 --- /dev/null +++ b/fawnds/cindex/semi_direct_16_absoff_bucketing.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include "common.hpp" +#include +#include + +namespace cindex +{ + class semi_direct_16_absoff_bucketing + { + public: + semi_direct_16_absoff_bucketing(std::size_t size = 0, std::size_t keys_per_bucket = 1, std::size_t keys_per_block = 1); + + void resize(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block); + + void insert(const std::size_t& index_offset, const std::size_t& dest_offset); + void finalize() {} + + std::size_t index_offset(std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + std::size_t dest_offset(std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + + std::size_t size() const CINDEX_WARN_UNUSED_RESULT; + + std::size_t bit_size() const CINDEX_WARN_UNUSED_RESULT; + + protected: + void store(const std::size_t& idx, const std::size_t& type, const std::size_t& mask, const std::size_t& shift, const std::size_t& v); + std::size_t load(const std::size_t& idx, const std::size_t& type, const std::size_t& mask, const std::size_t& shift) const; + + private: + std::size_t size_; + + std::vector > bucket_info_; + std::size_t current_i_; + + std::size_t last_index_offsets_[16]; + std::size_t last_dest_offsets_[16]; + + static const std::size_t ranks_[16]; + }; +} + diff --git a/fawnds/cindex/semi_direct_16_reloff_bucketing.cpp b/fawnds/cindex/semi_direct_16_reloff_bucketing.cpp new file mode 100644 index 0000000..aa86d35 --- /dev/null +++ b/fawnds/cindex/semi_direct_16_reloff_bucketing.cpp @@ -0,0 +1,239 @@ +#include "semi_direct_16_reloff_bucketing.hpp" +#include "expected_size.hpp" +#include +#include + +#define USE_KEY_DIVERGENCE_FOR_INDEX + +namespace cindex +{ + const std::size_t semi_direct_16_reloff_bucketing::ranks_[16] = { + 0 /* unused */, 0, 1, 2, + 0 /* unused */, 3, 4, 5, + 0 /* unused */, 6, 7, 8, + 0 /* unused */, 9, 10, 11 + }; + + semi_direct_16_reloff_bucketing::semi_direct_16_reloff_bucketing(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block) + { + resize(size, keys_per_bucket, keys_per_block); + assert(sizeof(std::size_t) == sizeof(long long int)); + } + + void + semi_direct_16_reloff_bucketing::resize(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block) + { + if (size % 16 != 0) + size = (size + 15) / 16 * 16; + size_ = size; + + keys_per_bucket_ = keys_per_bucket; + keys_per_block_ = keys_per_block; + bits_per_key_ = expected_size::index_size(keys_per_bucket_, keys_per_block_); + bits_per_bucket_ = static_cast(static_cast(keys_per_bucket_) * bits_per_key_); + if (size_) + { + std::cout << "expected_bits_per_key_trie_only: " << expected_size::index_size(keys_per_bucket_, keys_per_block_) << std::endl; + std::cout << "expected_index_offset_delta: " << bits_per_bucket_ << std::endl; + std::cout << "expected_dest_offset_delta: " << keys_per_bucket_ << std::endl; + } + + boost::array v; + v[0] = v[1] = 0u; + bucket_info_.resize(0); + bucket_info_.resize(size / 16 * 5, v); + current_i_ = 0; + + //if (size_) + // std::cout << "bucket_count: " << size << std::endl; + //if (keys_per_bucket_ > 256) + // std::cout << "warning: too high keys per bucket" << std::endl; + } + + void + semi_direct_16_reloff_bucketing::store(const std::size_t& idx, const std::size_t& type, const std::size_t& mask, const std::size_t& shift, const std::size_t& v) + { + assert((v & mask) == v); + (void)mask; + bucket_info_[idx][type] |= guarded_cast(v << shift); + } + + std::size_t + semi_direct_16_reloff_bucketing::load(const std::size_t& idx, const std::size_t& type, const std::size_t& mask, const std::size_t& shift) const + { + return (bucket_info_[idx][type] >> shift) & mask; + } + + void + semi_direct_16_reloff_bucketing::store_delta(const std::size_t& idx, const std::size_t& type, const std::size_t& mask, const std::size_t& shift, const std::size_t& id, const long long int& v) + { + long long int biased_v = v + guarded_cast(mask / 2); + if (biased_v < 0 || biased_v >= guarded_cast(mask) - 1) + { + // overflow + //std::cout << "overflow: id: " << id << " v: " << v << " biased_v: " << biased_v << std::endl; + assert(current_i_ < 1000 || overflow_.size() < current_i_ * 2 / 5); // allow up to 20 % + + std::pair ret = overflow_.insert(std::make_pair(id, guarded_cast(v))); + assert(ret.second); // no duplicate entry + store(idx, type, mask, shift, mask); + } + else + { + std::size_t stored_biased_v = guarded_cast(biased_v); + store(idx, type, mask, shift, stored_biased_v); + } + } + + long long int + semi_direct_16_reloff_bucketing::load_delta(const std::size_t& idx, const std::size_t& type, const std::size_t& mask, const std::size_t& shift, const std::size_t& id) const + { + std::size_t loaded_biased_v = load(idx, type, mask, shift); + if (loaded_biased_v == mask) + { + overflow_table::const_iterator it = overflow_.find(id); + assert(it != overflow_.end()); + return (*it).second; + } + else + return guarded_cast(loaded_biased_v) - guarded_cast(mask / 2); + } + + void + semi_direct_16_reloff_bucketing::insert(const std::size_t& index_offset, const std::size_t& dest_offset) + { + assert(current_i_ < size_); + + //std::cout << "i: " << current_i_ << std::endl; + //std::cout << "current: " << index_offset << " " << dest_offset << std::endl; + + std::size_t base_idx = current_i_ / 16 * 5; + if (current_i_ % 16 == 0) + { + store(base_idx + 0, 0, 0xffffffff, 0, index_offset); + store(base_idx + 0, 1, 0xffffffff, 0, dest_offset); + } + else if (current_i_ % 4 == 0) + { + std::size_t diff_base = current_i_ - 4; + long long int index_offset_delta = guarded_cast(index_offset - last_index_offsets_[diff_base % 16]) - guarded_cast(4 * bits_per_bucket_); + long long int dest_offset_delta = guarded_cast(dest_offset - last_dest_offsets_[diff_base % 16]) - guarded_cast(4 * keys_per_bucket_); + +#ifdef USE_KEY_DIVERGENCE_FOR_INDEX + index_offset_delta -= static_cast(static_cast(dest_offset_delta) * bits_per_key_); +#endif + + store_delta(base_idx + 1, 0, 0x3ff, ((current_i_ % 16) / 4 - 1) * 10, + current_i_ * 2 + 0, index_offset_delta); + store_delta(base_idx + 1, 1, 0x3ff, ((current_i_ % 16) / 4 - 1) * 10, + current_i_ * 2 + 1, dest_offset_delta); + } + else + { + std::size_t diff_base = current_i_ - 1; + long long int index_offset_delta = guarded_cast(index_offset - last_index_offsets_[diff_base % 16]) - guarded_cast(1 * bits_per_bucket_); + long long int dest_offset_delta = guarded_cast(dest_offset - last_dest_offsets_[diff_base % 16]) - guarded_cast(1 * keys_per_bucket_); + std::size_t rank = ranks_[current_i_ % 16]; + +#ifdef USE_KEY_DIVERGENCE_FOR_INDEX + index_offset_delta -= static_cast(static_cast(dest_offset_delta) * bits_per_key_); +#endif + + store_delta(base_idx + 2 + rank / 4, 0, 0xff, rank % 4 * 8, + current_i_ * 2 + 0, index_offset_delta); + store_delta(base_idx + 2 + rank / 4, 1, 0xff, rank % 4 * 8, + current_i_ * 2 + 1, dest_offset_delta); + } + + assert(this->index_offset(current_i_) == index_offset); + assert(this->dest_offset(current_i_) == dest_offset); + + last_index_offsets_[current_i_ % 16] = index_offset; + last_dest_offsets_[current_i_ % 16] = dest_offset; + + current_i_++; + } + + void + semi_direct_16_reloff_bucketing::finalize() + { + std::cout << "overflow: " << static_cast(overflow_.size()) / static_cast(current_i_ * 2) << std::endl; + } + + std::size_t + semi_direct_16_reloff_bucketing::index_offset(std::size_t i) const + { + assert(i < size_); + + std::size_t base_idx = i / 16 * 5; + std::size_t v; + if (i % 16 == 0) + v = load(base_idx + 0, 0, 0xffffffff, 0); + else if (i % 4 == 0) + { + std::size_t diff_base = i - 4; + v = load_delta(base_idx + 1, 0, 0x3ff, ((i % 16) / 4 - 1) * 10, i * 2 + 0) + index_offset(diff_base) + 4 * bits_per_bucket_; + +#ifdef USE_KEY_DIVERGENCE_FOR_INDEX + long long int dest_offset_delta = load_delta(base_idx + 1, 1, 0x3ff, ((i % 16) / 4 - 1) * 10, i * 2 + 0); + v += static_cast(static_cast(static_cast(dest_offset_delta) * bits_per_key_)); +#endif + } + else + { + std::size_t diff_base = i - 1; + std::size_t rank = ranks_[i % 16]; + v = load_delta(base_idx + 2 + rank / 4, 0, 0xff, rank % 4 * 8, i * 2 + 0) + index_offset(diff_base) + 1 * bits_per_bucket_; + +#ifdef USE_KEY_DIVERGENCE_FOR_INDEX + long long int dest_offset_delta = load_delta(base_idx + 2 + rank / 4, 1, 0xff, rank % 4 * 8, i * 2 + 1); + v += static_cast(static_cast(static_cast(dest_offset_delta) * bits_per_key_)); +#endif + } + + //std::cout << "index_offset(" << i << ") = " << v << std::endl; + return v; + } + + std::size_t + semi_direct_16_reloff_bucketing::dest_offset(std::size_t i) const + { + assert(i < size_); + + std::size_t base_idx = i / 16 * 5; + std::size_t v; + if (i % 16 == 0) + v = load(base_idx + 0, 1, 0xffffffff, 0); + else if (i % 4 == 0) + { + std::size_t diff_base = i - 4; + v = load_delta(base_idx + 1, 1, 0x3ff, ((i % 16) / 4 - 1) * 10, i * 2 + 1) + dest_offset(diff_base) + 4 * keys_per_bucket_; + } + else + { + std::size_t diff_base = i - 1; + std::size_t rank = ranks_[i % 16]; + v = load_delta(base_idx + 2 + rank / 4, 1, 0xff, rank % 4 * 8, i * 2 + 1) + dest_offset(diff_base) + 1 * keys_per_bucket_; + } + + //std::cout << "dest_offset(" << i << ") = " << v << std::endl; + return v; + } + + std::size_t + semi_direct_16_reloff_bucketing::size() const + { + return size_; + } + + std::size_t + semi_direct_16_reloff_bucketing::bit_size() const + { + std::size_t bit_size = bucket_info_.size() * 2 * sizeof(uint32_t) * 8; + // TODO: this assumes a non-chaining hash table with 50% utilization, + // which is not very true for most std::tr1::unordered_map implementations + bit_size += (sizeof(overflow_table::key_type) + sizeof(overflow_table::mapped_type)) * 8 * overflow_.size() * 2; + return bit_size; + } +} + diff --git a/fawnds/cindex/semi_direct_16_reloff_bucketing.hpp b/fawnds/cindex/semi_direct_16_reloff_bucketing.hpp new file mode 100644 index 0000000..63bfea6 --- /dev/null +++ b/fawnds/cindex/semi_direct_16_reloff_bucketing.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include "common.hpp" +#include +#include +#include + +namespace cindex +{ + class semi_direct_16_reloff_bucketing + { + public: + semi_direct_16_reloff_bucketing(std::size_t size = 0, std::size_t keys_per_bucket = 1, std::size_t keys_per_block = 1); + + void resize(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block); + + void insert(const std::size_t& index_offset, const std::size_t& dest_offset); + void finalize(); + + std::size_t index_offset(std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + std::size_t dest_offset(std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + + std::size_t size() const CINDEX_WARN_UNUSED_RESULT; + + std::size_t bit_size() const CINDEX_WARN_UNUSED_RESULT; + + protected: + void store(const std::size_t& idx, const std::size_t& type, const std::size_t& mask, const std::size_t& shift, const std::size_t& v); + std::size_t load(const std::size_t& idx, const std::size_t& type, const std::size_t& mask, const std::size_t& shift) const; + + void store_delta(const std::size_t& idx, const std::size_t& type, const std::size_t& mask, const std::size_t& shift, const std::size_t& id, const long long int& v); + long long int load_delta(const std::size_t& idx, const std::size_t& type, const std::size_t& mask, const std::size_t& shift, const std::size_t& id) const; + + private: + std::size_t size_; + std::size_t keys_per_bucket_; + std::size_t keys_per_block_; + + double bits_per_key_; + std::size_t bits_per_bucket_; + + std::vector > bucket_info_; + std::size_t current_i_; + + std::size_t last_index_offsets_[16]; + std::size_t last_dest_offsets_[16]; + + static const std::size_t ranks_[16]; + + typedef std::tr1::unordered_map overflow_table; + overflow_table overflow_; + }; +} + diff --git a/fawnds/cindex/sign_interleave.hpp b/fawnds/cindex/sign_interleave.hpp new file mode 100644 index 0000000..0059b89 --- /dev/null +++ b/fawnds/cindex/sign_interleave.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include "common.hpp" + +namespace cindex +{ + class sign_interleave + { + public: + template + static T encode(const T& v) + { + // 0, 1, 2, 3, ... => 0, 2, 4, 8, ... + // -1, -2, -3 -4, ... => 1, 3, 5, 7, ... + if ((v & (static_cast(1) << (sizeof(T) * 8 - 1))) == 0) + return static_cast(v << 1); + else + return static_cast(((~v) << 1) | 1); + } + + template + static T decode(const T& v) + { + // 0, 2, 4, 8, ... => 0, 1, 2, 3, ... + // 1, 3, 5, 7, ... => -1, -2, -3, -4, ... + if ((v & 1) == 0) + return static_cast((v >> 1) & ~(static_cast(1) << (sizeof(T) * 8 - 1))); + else + return static_cast(~(v >> 1) | (static_cast(1) << (sizeof(T) * 8 - 1))); + } + }; +} + diff --git a/fawnds/cindex/trie.hpp b/fawnds/cindex/trie.hpp new file mode 100644 index 0000000..d013af0 --- /dev/null +++ b/fawnds/cindex/trie.hpp @@ -0,0 +1,211 @@ +#pragma once + +#include "common.hpp" +#include "bit_access.hpp" +#include "huffman.hpp" +#include "exp_golomb.hpp" +#include "sign_interleave.hpp" +//#include + +namespace cindex +{ + template + class trie + { + public: + trie() + { + // prepare huffman coding for n <= HuffmanCodingLimit + for (unsigned int n = 2; n <= HuffmanCodingLimit; n++) + { + if (!WeakOrdering) + { + huffman_tree_generator gen(n + 1); + + uint64_t v = 1; + gen[0] = v; + for (unsigned int k = 1; k <= n; k++) + gen[k] = v = v * (n - k + 1) / k; + + huffman_tree t(n + 1); + gen.generate(t); + + huff_[n - 2] = new huffman(t); + } + else + { + huffman_tree_generator gen(n); + + uint64_t v = 1; + gen[0] = v * 2; + for (unsigned int k = 1; k <= n - 1; k++) + gen[k] = v = v * (n - k + 1) / k; + + huffman_tree t(n); + gen.generate(t); + + huff_[n - 2] = new huffman(t); + } + } + } + + template + void + recreate_huffman_from_dist(DistType& dist) + { + assert(!WeakOrdering); + + for (unsigned int n = 2; n <= HuffmanCodingLimit; n++) + delete huff_[n - 2]; + + for (unsigned int n = 2; n <= HuffmanCodingLimit; n++) + { + huffman_tree_generator gen(n + 1); + + for (unsigned int k = 0; k <= n; k++) + gen[k] = dist[n][k]; + + huffman_tree t(n + 1); + gen.generate(t); + + huff_[n - 2] = new huffman(t); + } + } + + virtual ~trie() + { + for (unsigned int n = 2; n <= HuffmanCodingLimit; n++) + { + delete huff_[n - 2]; + huff_[n - 2] = NULL; + } + } + + template + void encode(Buffer& out_buf, const KeyArrayType& arr, std::size_t key_len, std::size_t off, std::size_t n, std::size_t dest_base = 0, std::size_t dest_keys_per_block = 1, std::size_t skip_bits = 0) const + { + encode_rec(out_buf, arr, key_len, off, n, dest_base, dest_keys_per_block, skip_bits); + } + + template + std::size_t locate(const Buffer& in_buf, std::size_t& in_out_buf_iter, const uint8_t* key, std::size_t key_len, std::size_t off, std::size_t n, std::size_t dest_base = 0, std::size_t dest_keys_per_block = 1, std::size_t skip_bits = 0) const + { + return locate_rec(in_buf, in_out_buf_iter, key, key_len, off, n, dest_base, dest_keys_per_block, skip_bits); + } + + protected: + template + void + encode_rec(Buffer& out_buf, const KeyArrayType& arr, std::size_t key_len, std::size_t off, std::size_t n, std::size_t dest_base, std::size_t dest_keys_per_block, std::size_t depth) const + { + //std::cout << "encode: " << off << " " << n << " " << depth << std::endl; + + // do not encode 0- or 1-sized trees + /* + if (n == 1 || depth >= 4) + { + for (std::size_t k = depth; k < 4; k++) + out_buf.push_back(0); + return; + } + */ + if (n <= 1) + return; + + // k-perfect hashing + if (n <= dest_keys_per_block && (dest_base + off) / dest_keys_per_block == (dest_base + off + n - 1) / dest_keys_per_block) + return; + + assert(depth < key_len * 8); // duplicate key? + + // find the number of keys on the left tree + std::size_t left = 0; + for (; left < n; left++) + { + if (bit_access::get(arr[off + left], depth)) // assume sorted keys + break; + } + + // replace (n, 0) split with (0, n) split if weak ordering is used + if (WeakOrdering && left == n) + left = 0; + + // encode the left tree size + if (n <= HuffmanCodingLimit) + huff_[n - 2]->encode(out_buf, left); + else + exp_golomb<>::encode(out_buf, sign_interleave::encode(left - n / 2)); + + encode_rec(out_buf, arr, key_len, off, left, dest_base, dest_keys_per_block, depth + 1); + encode_rec(out_buf, arr, key_len, off + left, n - left, dest_base, dest_keys_per_block, depth + 1); + } + + template + std::size_t + locate_rec(const Buffer& in_buf, std::size_t& in_out_buf_iter, const uint8_t* key, std::size_t key_len, std::size_t off, std::size_t n, std::size_t dest_base, std::size_t dest_keys_per_block, std::size_t depth) const + { + //std::cout << "locate: " << off << " " << n << " " << depth << std::endl; + + // do not encode 0- or 1-sized trees + if (n <= 1) + return 0; + + // k-perfect hashing + if (n <= dest_keys_per_block && (dest_base + off) / dest_keys_per_block == (dest_base + off + n - 1) / dest_keys_per_block) + return 0; + + assert(depth < key_len * 8); // invalid code? + + // decode the left tree size + std::size_t left; + if (n <= HuffmanCodingLimit) + left = huff_[n - 2]->decode(in_buf, in_out_buf_iter); + else + left = sign_interleave::decode(exp_golomb<>::decode(in_buf, in_out_buf_iter)) + n / 2; + assert(left <= n); + + // find the number of keys on the left to the key (considering weak ordering) + if (!bit_access::get(key, depth) && (!WeakOrdering || (WeakOrdering && left != 0))) + { + return locate_rec(in_buf, in_out_buf_iter, key, key_len, off, left, dest_base, dest_keys_per_block, depth + 1); + } + else + { + skip_rec(in_buf, in_out_buf_iter, key, key_len, off, left, dest_base, dest_keys_per_block, depth + 1); + return left + locate_rec(in_buf, in_out_buf_iter, key, key_len, off + left, n - left, dest_base, dest_keys_per_block, depth + 1); + } + } + + template + void + skip_rec(const Buffer& in_buf, std::size_t& in_out_buf_iter, const uint8_t* key, std::size_t key_len, std::size_t off, std::size_t n, std::size_t dest_base, std::size_t dest_keys_per_block, std::size_t depth) const + { + //std::cout << "skip: " << off << " " << n << " " << depth << std::endl; + + // do not encode 0- or 1-sized trees + if (n <= 1) + return; + + // k-perfect hashing + if (n <= dest_keys_per_block && (dest_base + off) / dest_keys_per_block == (dest_base + off + n - 1) / dest_keys_per_block) + return; + + assert(depth < key_len * 8); // invalid code? + + // decode the left tree size + std::size_t left; + if (n <= HuffmanCodingLimit) + left = huff_[n - 2]->decode(in_buf, in_out_buf_iter); + else + left = sign_interleave::decode(exp_golomb<>::decode(in_buf, in_out_buf_iter)) + n / 2; + assert(left <= n); + + skip_rec(in_buf, in_out_buf_iter, key, key_len, off, left, dest_base, dest_keys_per_block, depth + 1); + skip_rec(in_buf, in_out_buf_iter, key, key_len, off + left, n - left, dest_base, dest_keys_per_block, depth + 1); + } + + protected: + huffman* huff_[HuffmanCodingLimit - 1]; + }; +} + diff --git a/fawnds/cindex/twolevel_absoff_bucketing.cpp b/fawnds/cindex/twolevel_absoff_bucketing.cpp new file mode 100644 index 0000000..85ef6a3 --- /dev/null +++ b/fawnds/cindex/twolevel_absoff_bucketing.cpp @@ -0,0 +1,111 @@ +#include "twolevel_absoff_bucketing.hpp" +#include + +namespace cindex +{ + template + twolevel_absoff_bucketing::twolevel_absoff_bucketing(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block) + { + resize(size, keys_per_bucket, keys_per_block); + } + + template + void + twolevel_absoff_bucketing::resize(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block CINDEX_UNUSED) + { + size_ = size; + keys_per_bucket_ = keys_per_bucket; + + std::size_t bits_per_key_bound = 4; + upper_bucket_size_ = (std::size_t(1) << (sizeof(ValueType) * 8)) / (bits_per_key_bound * keys_per_bucket_); + if (upper_bucket_size_ == 0) + upper_bucket_size_ = 1; + + bucket_info_.resize(size); + upper_bucket_info_.resize(size / upper_bucket_size_ + 2); + upper_bucket_info_[0][0] = 0; + upper_bucket_info_[0][1] = 0; + current_i_ = 0; + + //if (size_) + //{ + // std::cout << "bucket_count: " << size << std::endl; + // std::cout << "upper_bucket_count: " << size / upper_bucket_size_ + 2 << std::endl; + //} + } + + template + void + twolevel_absoff_bucketing::insert(const std::size_t& index_offset, const std::size_t& dest_offset) + { + assert(current_i_ < size_); + + std::size_t base_index_offset = upper_index_offset(current_i_ / upper_bucket_size_); + std::size_t base_dest_offset = upper_dest_offset(current_i_ / upper_bucket_size_); + + //std::cout << "i: " << current_i_ << std::endl; + //std::cout << "base: " << base_index_offset << " " << base_dest_offset << std::endl; + //std::cout << "current: " << index_offset << " " << dest_offset << std::endl; + + bucket_info_[current_i_][0] = guarded_cast(index_offset - base_index_offset); + bucket_info_[current_i_][1] = guarded_cast(dest_offset - base_dest_offset); + upper_bucket_info_[current_i_ / upper_bucket_size_ + 1][0] = guarded_cast(index_offset); + upper_bucket_info_[current_i_ / upper_bucket_size_ + 1][1] = guarded_cast(dest_offset); + + assert(this->index_offset(current_i_) == index_offset); + assert(this->dest_offset(current_i_) == dest_offset); + + current_i_++; + } + + template + std::size_t + twolevel_absoff_bucketing::index_offset(std::size_t i) const + { + assert(i < size_); + return upper_index_offset(i / upper_bucket_size_) + bucket_info_[i][0]; + } + + template + std::size_t + twolevel_absoff_bucketing::dest_offset(std::size_t i) const + { + assert(i < size_); + return upper_dest_offset(i / upper_bucket_size_) + bucket_info_[i][1]; + } + + template + std::size_t + twolevel_absoff_bucketing::upper_index_offset(std::size_t i) const + { + assert(i < size_ / upper_bucket_size_ + 1); + return upper_bucket_info_[i][0]; + } + + template + std::size_t + twolevel_absoff_bucketing::upper_dest_offset(std::size_t i) const + { + assert(i < size_ / upper_bucket_size_ + 1); + return upper_bucket_info_[i][1]; + } + + template + std::size_t + twolevel_absoff_bucketing::size() const + { + return size_; + } + + template + std::size_t + twolevel_absoff_bucketing::bit_size() const + { + std::size_t bit_size = bucket_info_.size() * 2 * sizeof(value_type) * 8; + bit_size += upper_bucket_info_.size() * 2 * sizeof(upper_value_type) * 8; + return bit_size; + } + + template class twolevel_absoff_bucketing<>; +} + diff --git a/fawnds/cindex/twolevel_absoff_bucketing.hpp b/fawnds/cindex/twolevel_absoff_bucketing.hpp new file mode 100644 index 0000000..a097baa --- /dev/null +++ b/fawnds/cindex/twolevel_absoff_bucketing.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include "common.hpp" +#include +#include + +namespace cindex +{ + template + class twolevel_absoff_bucketing + { + public: + typedef ValueType value_type; + typedef UpperValueType upper_value_type; + + public: + twolevel_absoff_bucketing(std::size_t size = 0, std::size_t keys_per_bucket = 1, std::size_t keys_per_block = 1); + + void resize(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block); + + void insert(const std::size_t& index_offset, const std::size_t& dest_offset); + void finalize() {} + + std::size_t index_offset(std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + std::size_t dest_offset(std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + + std::size_t size() const CINDEX_WARN_UNUSED_RESULT; + + std::size_t bit_size() const CINDEX_WARN_UNUSED_RESULT; + + protected: + std::size_t upper_index_offset(std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + std::size_t upper_dest_offset(std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + + private: + std::size_t size_; + std::size_t keys_per_bucket_; + + std::size_t upper_bucket_size_; + + std::vector > bucket_info_; + std::vector > upper_bucket_info_; + + std::size_t current_i_; + }; +} + diff --git a/fawnds/cindex/twolevel_reloff_bucketing.cpp b/fawnds/cindex/twolevel_reloff_bucketing.cpp new file mode 100644 index 0000000..a4f4a03 --- /dev/null +++ b/fawnds/cindex/twolevel_reloff_bucketing.cpp @@ -0,0 +1,255 @@ +#include "twolevel_reloff_bucketing.hpp" +#include "expected_size.hpp" +#include +#include + +#define USE_KEY_DIVERGENCE_FOR_INDEX + +namespace cindex +{ + twolevel_reloff_bucketing::twolevel_reloff_bucketing(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block) + { + resize(size, keys_per_bucket, keys_per_block); + assert(sizeof(std::size_t) == sizeof(long long int)); + } + + void + twolevel_reloff_bucketing::resize(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block) + { + if (size % group_size_ != 0) + size = (size + group_size_ - 1) / group_size_ * group_size_; + size_ = size; + + keys_per_bucket_ = keys_per_bucket; + keys_per_block_ = keys_per_block; + bits_per_key_ = expected_size::index_size(keys_per_bucket_, keys_per_block_); + bits_per_bucket_ = static_cast(static_cast(keys_per_bucket_) * bits_per_key_); + if (size_) + { + std::cout << "expected_bits_per_key_trie_only: " << expected_size::index_size(keys_per_bucket_, keys_per_block_) << std::endl; + std::cout << "expected_index_offset_delta: " << bits_per_bucket_ << std::endl; + std::cout << "expected_dest_offset_delta: " << keys_per_bucket_ << std::endl; + } + + boost::array v; + v[0] = v[1] = 0u; + bucket_info_.resize(0); + bucket_info_.resize(size_ / group_size_ * (4 + (group_size_ - 1)), v); + current_i_ = 0; + + //if (size_) + // std::cout << "bucket_count: " << size << std::endl; + //if (keys_per_bucket_ > 256) + // std::cout << "warning: too high keys per bucket" << std::endl; + + if (size_) + { + double c = 0.; + + if (bits_per_bucket_ > 0) + { + if (bits_per_bucket_ > 127) + c += gsl_cdf_poisson_P(guarded_cast(bits_per_bucket_) - 127, static_cast(bits_per_bucket_)); + c += (1. - gsl_cdf_poisson_P(guarded_cast(bits_per_bucket_) + 127, static_cast(bits_per_bucket_))); + } + + if (keys_per_bucket_ > 0) + { + if (keys_per_bucket_ > 127) + c += gsl_cdf_poisson_P(guarded_cast(keys_per_bucket_) - 127, static_cast(keys_per_bucket_)); + c += (1. - gsl_cdf_poisson_P(guarded_cast(keys_per_bucket_) + 127, static_cast(keys_per_bucket_))); + } + + std::cout << "expected_overflow: " << c / 2. << std::endl; + } + } + + void + twolevel_reloff_bucketing::store(const std::size_t& idx, const std::size_t& type, const std::size_t& v) + { + bucket_info_[idx][type] = guarded_cast(v); + } + + std::size_t + twolevel_reloff_bucketing::load(const std::size_t& idx, const std::size_t& type) const + { + return bucket_info_[idx][type]; + } + + void + twolevel_reloff_bucketing::store_delta(const std::size_t& idx, const std::size_t& type, const std::size_t& id, const long long int& v) + { + static const std::size_t mask = 0xff; + long long int biased_v = v + guarded_cast(mask / 2); + if (biased_v < 0 || biased_v >= guarded_cast(mask) - 1) + { + // overflow + //std::cout << "overflow: id: " << id << " v: " << v << " biased_v: " << biased_v << std::endl; + assert(current_i_ < 1000 || overflow_.size() < current_i_ * 2 / 5); // allow up to 20 % + + std::pair ret = overflow_.insert(std::make_pair(id, guarded_cast(v))); + assert(ret.second); // no duplicate entry + store(idx, type, mask); + } + else + { + std::size_t stored_biased_v = guarded_cast(biased_v); + store(idx, type, stored_biased_v); + } + } + + long long int + twolevel_reloff_bucketing::load_delta(const std::size_t& idx, const std::size_t& type, const std::size_t& id) const + { + static const std::size_t mask = 0xff; + std::size_t loaded_biased_v = load(idx, type); + if (loaded_biased_v == mask) + { + overflow_table::const_iterator it = overflow_.find(id); + assert(it != overflow_.end()); + return (*it).second; + } + else + return guarded_cast(loaded_biased_v) - guarded_cast(mask / 2); + } + + void + twolevel_reloff_bucketing::insert(const std::size_t& index_offset, const std::size_t& dest_offset) + { + assert(current_i_ < size_); + + //std::cout << "i: " << current_i_ << std::endl; + //std::cout << "current: " << index_offset << " " << dest_offset << std::endl; + + std::size_t base_idx = current_i_ / group_size_ * (4 + (group_size_ - 1)); + if (current_i_ % group_size_ == 0) + { + store(base_idx + 0, 0, (index_offset & 0xff000000) >> 24); + store(base_idx + 1, 0, (index_offset & 0x00ff0000) >> 16); + store(base_idx + 2, 0, (index_offset & 0x0000ff00) >> 8); + store(base_idx + 3, 0, (index_offset & 0x000000ff) >> 0); + + store(base_idx + 0, 1, (dest_offset & 0xff000000) >> 24); + store(base_idx + 1, 1, (dest_offset & 0x00ff0000) >> 16); + store(base_idx + 2, 1, (dest_offset & 0x0000ff00) >> 8); + store(base_idx + 3, 1, (dest_offset & 0x000000ff) >> 0); + } + else + { + std::size_t diff_base = current_i_ - 1; + long long int index_offset_delta = guarded_cast(index_offset - last_index_offsets_[diff_base % group_size_]) - guarded_cast(bits_per_bucket_); + long long int dest_offset_delta = guarded_cast(dest_offset - last_dest_offsets_[diff_base % group_size_]) - guarded_cast(keys_per_bucket_); + +#ifdef USE_KEY_DIVERGENCE_FOR_INDEX + index_offset_delta -= static_cast(static_cast(dest_offset_delta) * bits_per_key_); +#endif + + store_delta(base_idx + 4 + current_i_ % group_size_ - 1, 0, current_i_ * 2 + 0, index_offset_delta); + store_delta(base_idx + 4 + current_i_ % group_size_ - 1, 1, current_i_ * 2 + 1, dest_offset_delta); + } + + assert(this->index_offset(current_i_) == index_offset); + assert(this->dest_offset(current_i_) == dest_offset); + + last_index_offsets_[current_i_ % group_size_] = index_offset; + last_dest_offsets_[current_i_ % group_size_] = dest_offset; + + current_i_++; + } + + void + twolevel_reloff_bucketing::finalize() + { + std::cout << "overflow: " << static_cast(overflow_.size()) / static_cast(current_i_ * 2) << std::endl; + } + + std::size_t + twolevel_reloff_bucketing::index_offset(std::size_t i) const + { + assert(i < size_); + + std::size_t base_idx = i / group_size_ * (4 + (group_size_ - 1)); + std::size_t v; + + v = load(base_idx + 0, 0) << 24; + v |= load(base_idx + 1, 0) << 16; + v |= load(base_idx + 2, 0) << 8; + v |= load(base_idx + 3, 0) << 0; + + const std::size_t delta_idx_end = i % group_size_; + if (delta_idx_end != 0) + { + std::size_t load_idx = base_idx + 4; + std::size_t load_id = (i / group_size_ * group_size_ + 1) * 2 + 0; + + for (std::size_t delta_idx = 1; delta_idx <= delta_idx_end; delta_idx++) + { + v += load_delta(load_idx, 0, load_id); + +#ifdef USE_KEY_DIVERGENCE_FOR_INDEX + long long int dest_offset_delta = load_delta(load_idx, 1, load_id + 1); + v += static_cast(static_cast(static_cast(dest_offset_delta) * bits_per_key_)); +#endif + + load_idx++; + load_id += 2; + } + + v += (i % group_size_) * bits_per_bucket_; + } + + //std::cout << "index_offset(" << i << ") = " << v << std::endl; + return v; + } + + std::size_t + twolevel_reloff_bucketing::dest_offset(std::size_t i) const + { + assert(i < size_); + + std::size_t base_idx = i / group_size_ * (4 + (group_size_ - 1)); + std::size_t v; + + v = load(base_idx + 0, 1) << 24; + v |= load(base_idx + 1, 1) << 16; + v |= load(base_idx + 2, 1) << 8; + v |= load(base_idx + 3, 1) << 0; + + const std::size_t delta_idx_end = i % group_size_; + if (delta_idx_end != 0) + { + std::size_t load_idx = base_idx + 4; + std::size_t load_id = (i / group_size_ * group_size_ + 1) * 2 + 1; + + for (std::size_t delta_idx = 1; delta_idx <= delta_idx_end; delta_idx++) + { + v += load_delta(load_idx, 1, load_id); + + load_idx++; + load_id += 2; + } + + v += (i % group_size_) * keys_per_bucket_; + } + + //std::cout << "dest_offset(" << i << ") = " << v << std::endl; + return v; + } + + std::size_t + twolevel_reloff_bucketing::size() const + { + return size_; + } + + std::size_t + twolevel_reloff_bucketing::bit_size() const + { + std::size_t bit_size = bucket_info_.size() * 2 * sizeof(uint8_t) * 8; + // TODO: this assumes a non-chaining hash table with 50% utilization, + // which is not very true for most std::tr1::unordered_map implementations + bit_size += (sizeof(overflow_table::key_type) + sizeof(overflow_table::mapped_type)) * 8 * overflow_.size() * 2; + return bit_size; + } +} + diff --git a/fawnds/cindex/twolevel_reloff_bucketing.hpp b/fawnds/cindex/twolevel_reloff_bucketing.hpp new file mode 100644 index 0000000..b280926 --- /dev/null +++ b/fawnds/cindex/twolevel_reloff_bucketing.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include "common.hpp" +#include +#include +#include + +namespace cindex +{ + class twolevel_reloff_bucketing + { + public: + twolevel_reloff_bucketing(std::size_t size = 0, std::size_t keys_per_bucket = 1, std::size_t keys_per_block = 1); + + void resize(std::size_t size, std::size_t keys_per_bucket, std::size_t keys_per_block); + + void insert(const std::size_t& index_offset, const std::size_t& dest_offset); + void finalize(); + + std::size_t index_offset(std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + std::size_t dest_offset(std::size_t i) const CINDEX_WARN_UNUSED_RESULT; + + std::size_t size() const CINDEX_WARN_UNUSED_RESULT; + + std::size_t bit_size() const CINDEX_WARN_UNUSED_RESULT; + + protected: + void store(const std::size_t& idx, const std::size_t& type, const std::size_t& v); + std::size_t load(const std::size_t& idx, const std::size_t& type) const; + + void store_delta(const std::size_t& idx, const std::size_t& type, const std::size_t& id, const long long int& v); + long long int load_delta(const std::size_t& idx, const std::size_t& type, const std::size_t& id) const; + + private: + std::size_t size_; + std::size_t keys_per_bucket_; + std::size_t keys_per_block_; + + double bits_per_key_; + std::size_t bits_per_bucket_; + + std::vector > bucket_info_; + std::size_t current_i_; + + static const std::size_t group_size_ = 29; // for 64-byte alignment + //static const std::size_t group_size_ = 13; // for 32-byte alignment + + std::size_t last_index_offsets_[group_size_]; + std::size_t last_dest_offsets_[group_size_]; + + typedef std::tr1::unordered_map overflow_table; + overflow_table overflow_; + }; +} + diff --git a/fawnds/configuration.cc b/fawnds/configuration.cc new file mode 100644 index 0000000..d6d9e87 --- /dev/null +++ b/fawnds/configuration.cc @@ -0,0 +1,780 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ + +#include "configuration.h" +#include "debug.h" +#include "file_io.h" + +#include + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "DOMTreeErrorReporter.hpp" +#include "DOMWriteErrorHandler.hpp" + +using namespace std; + +namespace fawn { + + class ConfigurationMutex { + public: + static ConfigurationMutex* getInstance(); + pthread_mutex_t mutex_; + private: + ConfigurationMutex(); + static std::auto_ptr instance; + }; + + class ConfigurationObject { + public: + ConfigurationObject(); + ~ConfigurationObject(); + DOMDocument* doc_; + int instance_counter_; + mutable pthread_mutex_t config_mutex_; + }; + + + ConfigurationMutex::ConfigurationMutex() { + pthread_mutex_init(&mutex_, NULL); + } + + std::auto_ptr ConfigurationMutex::instance(NULL); + ConfigurationMutex* ConfigurationMutex::getInstance() { + if (instance.get() == NULL) { + instance.reset(new ConfigurationMutex()); + } + return instance.get(); + } + + ConfigurationObject::ConfigurationObject() + { + instance_counter_ = 1; + pthread_mutex_init(&config_mutex_, NULL); + } + + ConfigurationObject::~ConfigurationObject() + { + assert(instance_counter_ == 0); + pthread_mutex_destroy(&config_mutex_); + delete doc_; + } + + tbb::queuing_mutex Configuration::mutex_; + + Configuration::Configuration() + { + DPRINTF(2, "Configuration: Called constructor without config file\n"); + initialize(); + DPRINTF(2, "Configuration: Constructor: initialized!\n"); + XMLCh* XMLCh_name = XMLString::transcode("core"); + DOMImplementation * DOM_impl = DOMImplementationRegistry::getDOMImplementation(XMLCh_name); + XMLString::release(&XMLCh_name); + conf_obj_ = new ConfigurationObject(); + XMLCh_name = XMLString::transcode("fawnds"); + conf_obj_->doc_ = DOM_impl->createDocument(NULL, XMLCh_name, NULL); + XMLString::release(&XMLCh_name); + context_node_ = conf_obj_->doc_->getDocumentElement(); + DPRINTF(2, "Configuration: Constructor: created object!\n"); + } + + Configuration::Configuration(const string &config_file) + { + DPRINTF(2, "Configuration: Called constructor with config file: %s\n", config_file.c_str()); + initialize(); + DPRINTF(2, "Configuration: Constructor: initialized!\n"); + conf_obj_ = new ConfigurationObject(); + conf_obj_->doc_ = readConfigFile(config_file); + context_node_ = conf_obj_->doc_->getDocumentElement(); + + // handle + while (ExistsNode("child::include") == 0) { + char file[config_file.size() + 1024]; + strcpy(file, config_file.c_str()); + + string subconfig_file = string(dirname(file)) + "/"; + subconfig_file += GetStringValue("child::include/file"); + + Configuration* subconfig = new Configuration(subconfig_file); + + string source_gXPathExpression = GetStringValue("child::include/src"); + string dest_gXPathExpression = GetStringValue("child::include/dest"); + + DOMXPathResult* source_xpath_result = subconfig->getXPathResult(source_gXPathExpression); + DOMXPathResult* dest_xpath_result = this->getXPathResult(dest_gXPathExpression); + bool errorsOccured = false; + if (source_xpath_result != NULL && dest_xpath_result!= NULL) { + try { + DOMNode* source_node = source_xpath_result->getNodeValue(); + DOMNode* dest_node = dest_xpath_result->getNodeValue(); + dest_node->appendChild(conf_obj_->doc_->importNode(source_node, true)); + } + catch(const DOMXPathException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: CloneAndAppendNode: An error occurred during XML node value retrieval. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + + source_xpath_result->release(); + dest_xpath_result->release(); + } else { + errorsOccured = true; + } + + delete subconfig; + DeleteNode("child::include"); + } + + // handle + while (ExistsNode("child::set") == 0) { + string node = GetStringValue("child::set/node"); + string value = GetStringValue("child::set/value"); + SetStringValue(node, value); + DeleteNode("child::set"); + } + DPRINTF(2, "Configuration: Constructor: created object!\n"); + } + + Configuration::Configuration(const Configuration* const &other, bool copy) + { + if (!copy) { + DPRINTF(2, "Configuration: Called constructor with config object\n"); + initialize(); + DPRINTF(2, "Configuration: Constructor: initialized!\n"); + pthread_mutex_lock(&(ConfigurationMutex::getInstance()->mutex_)); + ++(other->conf_obj_->instance_counter_); + conf_obj_ = other->conf_obj_; + pthread_mutex_unlock(&(ConfigurationMutex::getInstance()->mutex_)); + context_node_ = other->context_node_; + DPRINTF(2, "Configuration: Constructor: created object!\n"); + } + else + { + DPRINTF(2, "Configuration: Called constructor with config object\n"); + initialize(); + DPRINTF(2, "Configuration: Constructor: initialized!\n"); + conf_obj_ = new ConfigurationObject(); + DOMImplementation* impl = DOMImplementation::getImplementation(); + assert(impl); + conf_obj_->doc_ = impl->createDocument(); // non-standard extension of Xerces-C++ + DOMNode* imported_node = conf_obj_->doc_->importNode(other->context_node_, true); + assert(imported_node); + conf_obj_->doc_->appendChild(imported_node); + context_node_ = conf_obj_->doc_->getDocumentElement(); + DPRINTF(2, "Configuration: Constructor: created object!\n"); + } + } + + Configuration::~Configuration() + { + pthread_mutex_lock(&(ConfigurationMutex::getInstance()->mutex_)); + --(conf_obj_->instance_counter_); + // TODO: this may have a race condition that deletes conf_obj_ while instance_counter_ was about to increase in the constructor. + // fixing this may require a weak pointer + if (conf_obj_->instance_counter_ <= 0) { + delete conf_obj_; + } + terminate(); + pthread_mutex_unlock(&(ConfigurationMutex::getInstance()->mutex_)); + } + + int + Configuration::CloneAndAppendNode(const string& source_gXPathExpression, const string& dest_gXPathExpression) + { + pthread_mutex_lock(&(conf_obj_->config_mutex_)); + DPRINTF(2, "Configuration: Called CloneAndAppendNode: source: %s dest: %s\n", source_gXPathExpression.c_str(), dest_gXPathExpression.c_str()); + DOMXPathResult* source_xpath_result = Configuration::getXPathResult(source_gXPathExpression); + DOMXPathResult* dest_xpath_result = Configuration::getXPathResult(dest_gXPathExpression); + bool errorsOccured = false; + if (source_xpath_result != NULL && dest_xpath_result!= NULL) { + try { + DOMNode* source_node = source_xpath_result->getNodeValue(); + DOMNode* dest_node = dest_xpath_result->getNodeValue(); + dest_node->appendChild(source_node->cloneNode(true)); + } + catch(const DOMXPathException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: CloneAndAppendNode: An error occurred during XML node value retrieval. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + + source_xpath_result->release(); + dest_xpath_result->release(); + } else { + errorsOccured = true; + } + + if(errorsOccured) { + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return -1; + } + + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return 0; + } + + Configuration* + Configuration::CloneAndAppendNodeAndGetConfig(const string& source_gXPathExpression, const string& dest_gXPathExpression) + { + pthread_mutex_lock(&(conf_obj_->config_mutex_)); + DPRINTF(2, "Configuration: Called CloneAndAppendNodeAndGetConfig: source: %s dest: %s\n", source_gXPathExpression.c_str(), dest_gXPathExpression.c_str()); + DOMXPathResult* source_xpath_result = Configuration::getXPathResult(source_gXPathExpression); + DOMXPathResult* dest_xpath_result = Configuration::getXPathResult(dest_gXPathExpression); + bool errorsOccured = false; + Configuration* ret = NULL; + if (source_xpath_result != NULL && dest_xpath_result!= NULL) { + try { + DOMNode* source_node = source_xpath_result->getNodeValue(); + DOMNode* dest_node = dest_xpath_result->getNodeValue(); + DOMNode* clone_node = source_node->cloneNode(true); + if (clone_node->getNodeType() != DOMNode::ELEMENT_NODE) { + errorsOccured = true; + } else { + dest_node->appendChild(clone_node); + ret = new Configuration(this); + ret->SetContextNode((DOMElement*)clone_node); + } + } + catch(const DOMXPathException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: CloneAndAppendNode: An error occurred during XML node value retrieval. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + + source_xpath_result->release(); + dest_xpath_result->release(); + } else { + errorsOccured = true; + } + + if(errorsOccured) { + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return NULL; + } + + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return ret; + } + + int + Configuration::CloneAndReplaceNode(const string& source_gXPathExpression, const string& dest_gXPathExpression) + { + pthread_mutex_lock(&(conf_obj_->config_mutex_)); + DPRINTF(2, "Configuration: Called CloneAndReplaceNode: source: %s dest: %s\n", source_gXPathExpression.c_str(), dest_gXPathExpression.c_str()); + DOMXPathResult* source_xpath_result = Configuration::getXPathResult(source_gXPathExpression); + DOMXPathResult* dest_xpath_result = Configuration::getXPathResult(dest_gXPathExpression); + bool errorsOccured = false; + if (source_xpath_result != NULL && dest_xpath_result!= NULL) { + try { + DOMNode* source_node = source_xpath_result->getNodeValue(); + DOMNode* dest_node = dest_xpath_result->getNodeValue(); + dest_node->getParentNode()->replaceChild(source_node->cloneNode(true), dest_node)->release(); + } + catch(const DOMXPathException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: CloneAndReplaceNode: An error occurred during XML node value retrieval. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + + source_xpath_result->release(); + dest_xpath_result->release(); + } else { + errorsOccured = true; + } + + if(errorsOccured) { + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return -1; + } + + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return 0; + } + + int + Configuration::CreateNodeAndAppend(const string& tagName, const string& dest_gXPathExpression) + { + pthread_mutex_lock(&(conf_obj_->config_mutex_)); + DPRINTF(2, "Configuration: Called CreateNodeAndAppend with path: %s and tagName: %s\n", dest_gXPathExpression.c_str(), tagName.c_str()); + DOMXPathResult* xpath_result = Configuration::getXPathResult(dest_gXPathExpression); + bool errorsOccured = false; + if (xpath_result != NULL) { + try { + DOMNode* node = xpath_result->getNodeValue(); + XMLCh* XMLCh_name = XMLString::transcode(tagName.c_str()); + DOMNode* new_node = conf_obj_->doc_->createElement(XMLCh_name); + XMLString::release(&XMLCh_name); + node->appendChild(new_node); + } + catch(const DOMXPathException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: CreateNodeAndAppend: An error occurred during XML node value retrieval. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + + xpath_result->release(); + } else { + errorsOccured = true; + } + + if(errorsOccured) { + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return -1; + } + + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return 0; + } + + int + Configuration::DeleteNode(const string& gXPathExpression) + { + pthread_mutex_lock(&(conf_obj_->config_mutex_)); + DPRINTF(2, "Configuration: Called DeleteNode with path: %s\n", gXPathExpression.c_str()); + DOMXPathResult* xpath_result = Configuration::getXPathResult(gXPathExpression); + bool errorsOccured = false; + if (xpath_result != NULL) { + try { + DOMNode* node = xpath_result->getNodeValue(); + if (node != NULL) { + node->getParentNode()->removeChild(node)->release(); + } else { + errorsOccured = true; + } + } + catch(const DOMXPathException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: DeleteNode: An error occurred during XML node value retrieval. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + } else { + errorsOccured = true; + } + + xpath_result->release(); + + if(errorsOccured) { + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return -1; + } + + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return 0; + } + + int + Configuration::ExistsNode(const string& gXPathExpression) const + { + pthread_mutex_lock(&(conf_obj_->config_mutex_)); + DPRINTF(2, "Configuration: Called ExistsNode with path: %s\n", gXPathExpression.c_str()); + DOMXPathResult* xpath_result = Configuration::getXPathResult(gXPathExpression); + bool errorsOccured = false; + if (xpath_result != NULL) { + try { + DOMNode* node = xpath_result->getNodeValue(); + if (node == NULL) { + errorsOccured = true; + } + } + catch(const DOMXPathException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: ExistsNode: An error occurred during XML node value retrieval. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + } else { + errorsOccured = true; + } + + xpath_result->release(); + + if(errorsOccured) { + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return -1; + } + + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return 0; + + } + + const string + Configuration::GetStringValue(const string& gXPathExpression) const + { + pthread_mutex_lock(&(conf_obj_->config_mutex_)); + DPRINTF(2, "Configuration: Called getStringValue with path: %s\n", gXPathExpression.c_str()); + bool errorsOccured = false; + string result; + DOMXPathResult* xpath_result = Configuration::getXPathResult(gXPathExpression); + if (xpath_result != NULL) { + try { + DOMNode* node = xpath_result->getNodeValue(); + if (node == NULL) { + errorsOccured = true; + } else { + const XMLCh* XMLCh_result = node->getTextContent(); + char* result_c = XMLString::transcode(XMLCh_result); + result.append(result_c); + XMLString::release(&result_c); + } + } + catch(const DOMXPathException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: GetStringValue: An error occurred during XML node value retrieval. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + + xpath_result->release(); + } else + errorsOccured = true; + + DPRINTF(2, "Configuration: getStringValue: result: %s\n", tmp.c_str()); + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return result; + } + + int + Configuration::SetStringValue(const string& gXPathExpression, const string& value) + { + pthread_mutex_lock(&(conf_obj_->config_mutex_)); + DPRINTF(2, "Configuration: Called setStringValue with path: %s and value: %s\n", gXPathExpression.c_str(), value.c_str()); + DOMXPathResult* xpath_result = Configuration::getXPathResult(gXPathExpression); + bool errorsOccured = false; + try { + DOMNode* node = xpath_result->getNodeValue(); + if (!node) { + XERCES_STD_QUALIFIER cerr << "Configuration: SetStringValue: Node for " << gXPathExpression << " not found" + << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + XMLCh* XMLCh_value = XMLString::transcode(value.c_str()); + node->setTextContent(XMLCh_value); + XMLString::release(&XMLCh_value); + } + catch(const DOMXPathException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: SetStringValue: An error occurred during XML node value retrieval. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + + xpath_result->release(); + + if(errorsOccured) { + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return -1; + } + + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return 0; + } + + int + Configuration::WriteConfigFile(const std::string& config_file) const + { + pthread_mutex_lock(&(conf_obj_->config_mutex_)); + bool errorsOccured = false; + try { + // get a serializer, an instance of DOMLSSerializer + XMLCh tempStr[3] = {chLatin_L, chLatin_S, chNull}; + DOMImplementation *impl = DOMImplementationRegistry::getDOMImplementation(tempStr); + DOMLSSerializer *theSerializer = ((DOMImplementationLS*)impl)->createLSSerializer(); + DOMLSOutput *theOutputDesc = ((DOMImplementationLS*)impl)->createLSOutput(); + + // plug in user's own error handler + DOMErrorHandler *myErrorHandler = new DOMWriteErrorHandler(); + DOMConfiguration* serializerConfig = theSerializer->getDomConfig(); + serializerConfig->setParameter(XMLUni::fgDOMErrorHandler, myErrorHandler); + + // // set feature if the serializer supports the feature/mode + // if (serializerConfig->canSetParameter(XMLUni::fgDOMWRTSplitCdataSections, gSplitCdataSections)) + // serializerConfig->setParameter(XMLUni::fgDOMWRTSplitCdataSections, gSplitCdataSections); + + // if (serializerConfig->canSetParameter(XMLUni::fgDOMWRTDiscardDefaultContent, gDiscardDefaultContent)) + // serializerConfig->setParameter(XMLUni::fgDOMWRTDiscardDefaultContent, gDiscardDefaultContent); + + // if (serializerConfig->canSetParameter(XMLUni::fgDOMWRTFormatPrettyPrint, gFormatPrettyPrint)) + // serializerConfig->setParameter(XMLUni::fgDOMWRTFormatPrettyPrint, gFormatPrettyPrint); + + // if (serializerConfig->canSetParameter(XMLUni::fgDOMWRTBOM, gWriteBOM)) + // serializerConfig->setParameter(XMLUni::fgDOMWRTBOM, gWriteBOM); + + // + // Plug in a format target to receive the resultant + // XML stream from the serializer. + // + // StdOutFormatTarget prints the resultant XML stream + // to stdout once it receives any thing from the serializer. + // + XMLFormatTarget *myFormTarget; + myFormTarget = new LocalFileFormatTarget(config_file.c_str()); + theOutputDesc->setByteStream(myFormTarget); + + theSerializer->write(conf_obj_->doc_, theOutputDesc); + + theOutputDesc->release(); + theSerializer->release(); + + // + // Filter, formatTarget and error handler + // are NOT owned by the serializer. + // + if(myFormTarget) + delete myFormTarget; + if(myErrorHandler) + delete myErrorHandler; + + // if (gUseFilter) + // delete myFilter; + + } + catch (const OutOfMemoryException&) { + XERCES_STD_QUALIFIER cerr << "Configuration: WriteConfigFile: OutOfMemoryException" << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + catch (XMLException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: WriteConfigFile: An error occurred during creation of output transcoder. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + + if(errorsOccured) { + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return -1; + } + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return 0; + } + + int + Configuration::SetContextNode(const string& gXPathExpression) + { + pthread_mutex_lock(&(conf_obj_->config_mutex_)); + DPRINTF(2, "Configuration: Called SetContextNode with path: %s\n", gXPathExpression.c_str()); + DOMXPathResult* xpath_result = Configuration::getXPathResult(gXPathExpression); + bool errorsOccured = false; + if (xpath_result != NULL) { + try { + DPRINTF(2, "Configuration: SetContextNode: test result isNode\n"); + if (xpath_result->getResultType() == DOMXPathResult::FIRST_ORDERED_NODE_TYPE) { + DPRINTF(2, "Configuration: SetContextNode: try to retrieve node from result\n"); + DOMNode* node = xpath_result->getNodeValue(); + if (node != NULL) { + if (node->getNodeType() != DOMNode::ELEMENT_NODE) { + DPRINTF(2, "Configuration: SetContextNode: result node was not of type DOMElement (=1) but of type: %i\n", node->getNodeType()); + errorsOccured = true; + } else { + DPRINTF(2, "Configuration: SetContextNode: node value inside result is null or not of correct type\n"); + context_node_ = (DOMElement*)node; + } + } else { + DPRINTF(2, "Configuration: SetContextNode: result node is null (no node found)\n"); + errorsOccured = true; + } + } else { + DPRINTF(2, "Configuration: SetContextNode: result not of type first_ordered_node_type\n"); + errorsOccured = true; + } + } + catch(const DOMXPathException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: SetContextNode: An error occurred during XML node value retrieval. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + + xpath_result->release(); + } else { + DPRINTF(2, "Configuration: SetContextNode: xPathResult is null\n"); + errorsOccured = true; + } + if(errorsOccured) { + DPRINTF(2, "Configuration: SetContextNode: errors occured\n"); + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return -1; + } + + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + return 0; + } + + void + Configuration::ResetContextNode() + { + pthread_mutex_lock(&(conf_obj_->config_mutex_)); + context_node_ = conf_obj_->doc_->getDocumentElement(); + pthread_mutex_unlock(&(conf_obj_->config_mutex_)); + } + + + DOMDocument* + Configuration::readConfigFile(const string& config_file) + { + // + // Create our parser, then attach an error handler to the parser. + // The parser will call back to methods of the ErrorHandler if it + // discovers errors during the course of parsing the XML document. + // + XercesDOMParser *parser = new XercesDOMParser; + // parser->setValidationScheme(gValScheme); + // parser->setDoNamespaces(gDoNamespaces); + // parser->setDoSchema(gDoSchema); + // parser->setHandleMultipleImports (true); + // parser->setValidationSchemaFullChecking(gSchemaFullChecking); + // parser->setCreateEntityReferenceNodes(gDoCreate); + + DOMTreeErrorReporter *errReporter = new DOMTreeErrorReporter(); + parser->setErrorHandler(errReporter); + + // + // Parse the XML file, catching any XML exceptions that might propogate + // out of it. + // + bool errorsOccured = false; + try { + parser->parse(config_file.c_str()); + } + catch (const OutOfMemoryException&) { + XERCES_STD_QUALIFIER cerr << "Configuration: readConfigFile: OutOfMemoryException" << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + catch (const XMLException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: readConfigFile: An error occurred during parsing\n Message: " + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + catch (const DOMException& e) { + const unsigned int maxChars = 2047; + XMLCh errText[maxChars + 1]; + + XERCES_STD_QUALIFIER cerr << "\nConfiguration: readConfigFile: DOM Error during parsing: '" << config_file << "'\n" + << "DOMException code is: " << e.code << XERCES_STD_QUALIFIER endl; + + if (DOMImplementation::loadDOMExceptionMsg(e.code, errText, maxChars)) + XERCES_STD_QUALIFIER cerr << "Message is: " << StrX(errText) << XERCES_STD_QUALIFIER endl; + + errorsOccured = true; + } + + catch (...) { + XERCES_STD_QUALIFIER cerr << "Configuration: readConfigFile: An error occurred during parsing\n " << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + + DOMDocument *doc = NULL; + + // If the parse was successful, output the document data from the DOM tree + if (!errorsOccured && !errReporter->getSawErrors()) { + // get the DOM representation + doc = parser->getDocument(); + parser->adoptDocument(); + } + + // + // Clean up the error handler. The parser does not adopt handlers + // since they could be many objects or one object installed for multiple + // handlers. + // + if (errReporter) + delete errReporter; + + // + // Delete the parser itself. Must be done prior to calling Configuration::Terminate, below. + // + if (parser) + delete parser; + + return doc; + } + + DOMXPathResult* + Configuration::getXPathResult(const string& gXPathExpression) const + { + bool errorsOccured = false; + DOMXPathResult* result = NULL; + if(gXPathExpression != "") + { + XMLCh* xpathStr=XMLString::transcode(gXPathExpression.c_str()); + try { + result = conf_obj_->doc_->evaluate(xpathStr, + context_node_, + NULL, + DOMXPathResult::FIRST_ORDERED_NODE_TYPE, + NULL); + } + catch(const DOMXPathException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: getXPathResult: An error occurred during processing of the XPath expression. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + catch(const DOMException& e) { + XERCES_STD_QUALIFIER cerr << "Configuration: getXPathResult: An error occurred during processing of the XPath expression. Msg is:" + << XERCES_STD_QUALIFIER endl + << StrX(e.getMessage()) << XERCES_STD_QUALIFIER endl; + errorsOccured = true; + } + XMLString::release(&xpathStr); + } + else + errorsOccured = true; + + if(errorsOccured) { + DPRINTF(2, "Configuration: getXPathResult: errors occured\n"); + return NULL; + } + + return result; + } + + void + Configuration::SetContextNode(DOMElement* node) { + context_node_ = node; + } + + void Configuration::initialize() + { + // Configuration::Initialize the XML4C2 system + // this requires locking according to http://xerces.apache.org/xerces-c/faq-parse-3.html#faq-6 + + tbb::queuing_mutex::scoped_lock lock(mutex_); + + try { + XMLPlatformUtils::Initialize(); + } + + catch(const XMLException &toCatch) { + XERCES_STD_QUALIFIER cerr << "Configuration: initialize: Error during Xerces-c Initialization.\n" + << " Exception message:" + << StrX(toCatch.getMessage()) << XERCES_STD_QUALIFIER endl; + exit(-1); + } + } + + void Configuration::terminate() + { + tbb::queuing_mutex::scoped_lock lock(mutex_); + + // And call the termination method + XMLPlatformUtils::Terminate(); + } + +} // namespace fawn diff --git a/fawnds/configuration.h b/fawnds/configuration.h new file mode 100644 index 0000000..96cca89 --- /dev/null +++ b/fawnds/configuration.h @@ -0,0 +1,53 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _CONFIGURATION_H_ +#define _CONFIGURATION_H_ + +#include +#include +#include + +XERCES_CPP_NAMESPACE_BEGIN + class DOMDocument; + class DOMXPathResult; + class DOMElement; +XERCES_CPP_NAMESPACE_END + +namespace fawn { + class ConfigurationObject; + + class Configuration { + public: + Configuration(); + Configuration(const std::string &config_file); + Configuration(const Configuration* const &other, bool copy = false); + ~Configuration(); + + int CloneAndAppendNode(const std::string& source_gXPathExpression, const std::string& dest_gXPathExpression); + Configuration* CloneAndAppendNodeAndGetConfig(const std::string& source_gXPathExpression, const std::string& dest_gXPathExpression); + int CloneAndReplaceNode(const std::string& source_gXPathExpression, const std::string& dest_gXPathExpression); + int CreateNodeAndAppend(const std::string& tagName, const std::string& dest_gXPathExpression); + int DeleteNode(const std::string& gXPathExpression); + int ExistsNode(const std::string& gXPathExpression) const; + const std::string GetStringValue(const std::string& gXPathExpression) const; + int SetStringValue(const std::string& gXPathExpression, const std::string& value); + int WriteConfigFile(const std::string& config_file) const; + int SetContextNode(const std::string& gXPathExpression); + void ResetContextNode(); + private: + XERCES_CPP_NAMESPACE_QUALIFIER DOMDocument* readConfigFile(const std::string& config_file); + XERCES_CPP_NAMESPACE_QUALIFIER DOMXPathResult* getXPathResult(const std::string& gXPathExpression) const; + + void SetContextNode(XERCES_CPP_NAMESPACE_QUALIFIER DOMElement* node); + + void initialize(); + void terminate(); + + ConfigurationObject* conf_obj_; + XERCES_CPP_NAMESPACE_QUALIFIER DOMElement* context_node_; + + static tbb::queuing_mutex mutex_; + }; + +} // namespace fawn + +#endif // #ifndef _CONFIGURATION_H_ diff --git a/fawnds/fawnds.cc b/fawnds/fawnds.cc new file mode 100644 index 0000000..eec4539 --- /dev/null +++ b/fawnds/fawnds.cc @@ -0,0 +1,151 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "fawnds.h" +#include "configuration.h" +#include + +namespace fawn { + + FawnDS::FawnDS() + : config_(NULL) + { + } + + FawnDS::~FawnDS() + { + delete config_; + config_ = NULL; + } + + FawnDS_Return + FawnDS::SetConfig(const Configuration* config) + { + if (config_) + return ERROR; + config_ = config; + return OK; + } + + const Configuration* + FawnDS::GetConfig() const + { + return config_; + } + + FawnDS_Return + FawnDS::Create() + { + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS::Open() + { + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS::ConvertTo(FawnDS* new_store) const + { + (void)new_store; + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS::Flush() + { + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS::Close() + { + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS::Destroy() + { + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS::Status(const FawnDS_StatusType& type, Value& status) const + { + (void)type; (void)status; + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS::Put(const ConstValue& key, const ConstValue& data) + { + (void)key; (void)data; + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS::Append(Value& key, const ConstValue& data) + { + (void)key; (void)data; + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS::Delete(const ConstValue& key) + { + (void)key; + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS::Contains(const ConstValue& key) const + { + (void)key; + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS::Length(const ConstValue& key, size_t& len) const + { + (void)key; (void)len; + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS::Get(const ConstValue& key, Value& data, size_t offset, size_t len) const + { + (void)key; (void)data; (void)offset; (void)len; + return UNSUPPORTED; + } + + FawnDS_ConstIterator + FawnDS::Enumerate() const + { + std::cerr << "unsupported Enumerate() called" << std::endl; + return FawnDS_ConstIterator(); + } + + FawnDS_Iterator + FawnDS::Enumerate() + { + std::cerr << "unsupported Enumerate() called" << std::endl; + return FawnDS_Iterator(); + } + + FawnDS_ConstIterator + FawnDS::Find(const ConstValue& key) const + { + std::cerr << "unsupported Find() called" << std::endl; + (void)key; + return FawnDS_ConstIterator(); + } + + FawnDS_Iterator + FawnDS::Find(const ConstValue& key) + { + std::cerr << "unsupported Find() called" << std::endl; + (void)key; + return FawnDS_Iterator(); + } + +} // namespace fawn + diff --git a/fawnds/fawnds.h b/fawnds/fawnds.h new file mode 100644 index 0000000..bd8bd2d --- /dev/null +++ b/fawnds/fawnds.h @@ -0,0 +1,56 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _FAWNDS_H_ +#define _FAWNDS_H_ + +#include "basic_types.h" +#include "fawnds_types.h" +#include "value.h" +#include "fawnds_iterator.h" + +namespace fawn { + + class Configuration; + + // common interface for FAWN-DS stores + class FawnDS { + public: + FawnDS(); + virtual ~FawnDS(); + + FawnDS_Return SetConfig(const Configuration* config); // move + const Configuration* GetConfig() const; + + virtual FawnDS_Return Create(); + virtual FawnDS_Return Open(); + + virtual FawnDS_Return ConvertTo(FawnDS* new_store) const; + + virtual FawnDS_Return Flush(); + virtual FawnDS_Return Close(); + + virtual FawnDS_Return Destroy(); + + virtual FawnDS_Return Status(const FawnDS_StatusType& type, Value& status) const; + + virtual FawnDS_Return Put(const ConstValue& key, const ConstValue& data); + virtual FawnDS_Return Append(Value& key, const ConstValue& data); + + virtual FawnDS_Return Delete(const ConstValue& key); + + virtual FawnDS_Return Contains(const ConstValue& key) const; + virtual FawnDS_Return Length(const ConstValue& key, size_t& len) const; + virtual FawnDS_Return Get(const ConstValue& key, Value& data, size_t offset = 0, size_t len = -1) const; + + virtual FawnDS_ConstIterator Enumerate() const; + virtual FawnDS_Iterator Enumerate(); + + virtual FawnDS_ConstIterator Find(const ConstValue& key) const; + virtual FawnDS_Iterator Find(const ConstValue& key); + + protected: + const Configuration* config_; + }; + +} // namespace fawn + +#endif // #ifndef _FAWNDS_H_ diff --git a/fawnds/fawnds_bench.cc b/fawnds/fawnds_bench.cc new file mode 100644 index 0000000..25db128 --- /dev/null +++ b/fawnds/fawnds_bench.cc @@ -0,0 +1,332 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "fawnds_factory.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "print.h" +#include "dbid.h" +#include "hashutil.h" +#include "timing.h" + +enum { SCAN_RANDOM, SCAN_SEQUENTIAL }; + +using namespace std; +using namespace fawn; +using fawn::HashUtil; + +struct benchData { + FawnDS** f; + u_int num_records; + u_int num_to_scan; + u_int offset; + u_int numThreads; +}; + +volatile uint32_t readyCount = 0; +pthread_mutex_t count_lock; +vector search_times; +u_int max_record = 0; + + +void +usage() +{ + fprintf(stderr, + "fawnds_bench [-hfsw] [-r num_scan] [-n num_rec] [-c \"config1 config2 config3\"] [-a num_procs]\n" + ); +} + +void +help() +{ + usage(); + fprintf(stderr, + " -h help (this text)\n" + " -f fill\n" + " -s sequential scan\n" + " -r # number of entries to randomly scan\n" + " -n # number of entries to fill or scan.\n" + " -c # config files for one or more threads\n" + " -a # Turn on thread affinity, specify number of processors\n" + " -w random write test\n" + " -S # set value size (bytes)\n" + ); +} + +void *randomReadThread(void* p) +{ + benchData* bd = (benchData*)p; + FawnDS** dbs = bd->f; + u_int num_records = bd->num_records; + u_int num_to_scan = bd->num_to_scan; + u_int offsetVal = bd->offset; + FawnDS *mydb = dbs[offsetVal]; + + struct timeval tv_start, tv_end; + char *l = (char *)malloc(sizeof(char) * num_to_scan * sizeof(uint32_t)); + + for (u_int i = 0; i < num_to_scan; i++) { + u_int val = (offsetVal * num_records) + (rand()%num_records); + if (val < max_record) { + string ps_key((const char *)&val, sizeof(val)); + uint32_t key_id = HashUtil::BobHash(ps_key); + DBID key((char *)&key_id, sizeof(u_int32_t)); + memcpy(l + (i * sizeof(uint32_t)), key.data(), key.get_actual_size()); + } + else { + i--; + } + } + + pthread_mutex_lock(&count_lock); + readyCount++; + pthread_mutex_unlock(&count_lock); + struct timespec req; + req.tv_sec = 0; + req.tv_nsec = 20000; + + while (readyCount < bd->numThreads) { + nanosleep(&req, NULL); + } + gettimeofday(&tv_start, NULL); + //char* data; + //uint32_t data_len; + //std::string data; + Value data; + const char *key = l; + //bool callerDeletes = true; + for (u_int i = 0; i < num_to_scan; ++i) { + if(mydb->Get(ConstRefValue(key, sizeof(uint32_t)), data) != OK) { + perror("Get failed.\n"); + } + key += sizeof(uint32_t); + } + + gettimeofday(&tv_end, NULL); + double dbsearch_time = timeval_diff(&tv_start, &tv_end); + //printf("Time to search DB: %f seconds\n", dbsearch_time); + //printf("Query rate: %f\n", ((double)num_to_scan / dbsearch_time) ); + free(l); + + pthread_mutex_lock(&count_lock); + search_times.push_back(dbsearch_time); + pthread_mutex_unlock(&count_lock); + return NULL; +} + +void bench(int argc, char** argv) { + extern char *optarg; + extern int optind; + + int ch; + vector fileBases; + + u_int num_to_scan = 0; + int mode = SCAN_RANDOM; + bool createdb = false; + int writeTest = 0; + int numThreads = 1; + bool useThreads = false; + bool setAffinity = false; + int numProcs = 1; + int valuesize = 1024; + + while ((ch = getopt(argc, argv, "hfn:r:p:swc:a:S:")) != -1) + switch (ch) { + case 'n': + max_record = atoi(optarg); + break; + case 'r': + num_to_scan = atoi(optarg); + break; + case 'f': + createdb = true; + break; + case 's': + mode = SCAN_SEQUENTIAL; + break; + case 'w': + writeTest = 1; + break; + case 'c': + tokenize(optarg, fileBases, " "); + useThreads = true; + break; + case 'S': + valuesize = atoi(optarg); + break; + case 'a': + setAffinity = true; + numProcs = atoi(optarg); + break; + case 'h': + help(); + exit(0); + default: + usage(); + exit(-1); + } + argc -= optind; + argv += optind; + + if (fileBases.size() == 0 || argc != 0) { + usage(); + exit(-1); + } + + numThreads = fileBases.size(); + + struct timeval tv_start, tv_end; + gettimeofday(&tv_start, NULL); + + //char key[4]; + //char data[valuesize]; + //string value(valuesize, 'a'); + + FawnDS **dbs = (FawnDS**)malloc(numThreads * sizeof(FawnDS*)); + + // size? num_records / numThreads + int num_recs_per_db = (int) (max_record / numThreads); + + int bucket = num_recs_per_db; + if (max_record % numThreads != 0) + bucket += 1; + + pthread_t* workerThreadIds_ = (pthread_t*) malloc (numThreads * sizeof(pthread_t)); + benchData* bd = (benchData*) malloc (numThreads * sizeof(benchData)); + + //u_int fileBaseOffset = 0; + for (u_int i = 0; i < fileBases.size(); i++) { + ostringstream dbname_i; + dbname_i << fileBases[i]; + if (createdb) { + + + //TODO: Put size value inside config file! + + + // uint64_t size = num_recs_per_db * 2; + dbs[i] = FawnDS_Factory::New(dbname_i.str().c_str()); // .9, .8, htstore + dbs[i]->Create(); + } else { + printf("opening fawnds with config file %s\n", dbname_i.str().c_str()); + dbs[i] = FawnDS_Factory::New(dbname_i.str().c_str()); // htstore + dbs[i]->Open(); + } + } + + if (createdb && !writeTest) { + // Fill it sequentially if we're not testing writing + for (u_int i = 0; i < max_record; ++i) { + int num = i; + string ps_key((const char *)&num, sizeof(num)); + u_int32_t key_id = HashUtil::BobHash(ps_key); + DBID key((char *)&key_id, sizeof(u_int32_t)); + + int dbi = (int)(i / bucket); + + Value data; + if(dbs[dbi]->Put(ConstRefValue(key.data(), sizeof(uint32_t)), data) != OK) { + perror("Insert failed\n"); + } + } + + // this is required since we're not splitting/merging/rewriting initially + for (int i = 0; i < numThreads; i++) { + if (dbs[i]->Flush()) { + perror("Could not flush FawnDS.\n"); + } + } + } + gettimeofday(&tv_end, NULL); + + if (createdb) { + double dbcreate_time = timeval_diff(&tv_start, &tv_end); + printf("Time to create DB: %f seconds\n", dbcreate_time); + } + + srand((tv_end.tv_sec << 2) + tv_end.tv_usec); + + // Set of randomly ordered test elements + + //random_shuffle(l.begin(), l.end()); + + if (writeTest) { + vector l; + for (u_int i = 0; i < num_to_scan; i++) { + l.push_back(rand()%max_record); + } + int n = l.size(); + for (int i = 0; i < n; i++) { + u_int val = l[i]; + string ps_key((const char *)&val, sizeof(val)); + u_int32_t key_id = HashUtil::BobHash(ps_key); + DBID key((char *)&key_id, sizeof(u_int32_t)); + + int dbi = (int)(i / bucket); + Value data; + if(dbs[dbi]->Get(ConstRefValue(key.data(), sizeof(uint32_t)), data) != OK) { + perror("Insert failed\n"); + } + } + + } else { + pthread_mutex_init(&count_lock, NULL); + for (int i = 0; i < numThreads; i++) { + bd[i].f = dbs; + bd[i].num_to_scan = num_to_scan; + bd[i].num_records = bucket; + bd[i].offset = i; + bd[i].numThreads = numThreads; + if (useThreads) { + pthread_attr_t attr; + pthread_attr_init(&attr); +#ifdef cpu_set_t /* GNU/Linux-only! */ + if (setAffinity) { + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(i % numProcs, &cpuset); + pthread_attr_setaffinity_np(&attr, sizeof(cpuset), &cpuset); + } +#endif + pthread_create(&workerThreadIds_[i], &attr, + randomReadThread, &bd[i]); + } else { + randomReadThread(&bd[0]); + } + } + } + + if (useThreads) { + for (int i = 0; i < numThreads; i++) { + pthread_join(workerThreadIds_[i], NULL); + } + } + + pthread_mutex_destroy(&count_lock); + free(workerThreadIds_); + free(bd); + + double totalTime = 0; + for (int i = 0; i < numThreads; i++) { + totalTime = max(totalTime, search_times[i]); + } + double totalQueries = num_to_scan * numThreads; + + cout << "Aggregate Query Rate: " << totalQueries/totalTime << " queries per second" << endl; +} + + +int main(int argc, char** argv) { + bench(argc, argv); + return 0; +} diff --git a/fawnds/fawnds_combi.cc b/fawnds/fawnds_combi.cc new file mode 100644 index 0000000..b1396e4 --- /dev/null +++ b/fawnds/fawnds_combi.cc @@ -0,0 +1,1133 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "fawnds_combi.h" +#include "fawnds_factory.h" +#include "debug.h" +#include "global_limits.h" +#include +//#include "print.h" + +#include +#include +#include +#include + +namespace fawn { + + FawnDS_Combi::FawnDS_Combi() + : open_(false) + { + for (size_t stage = 0; stage < 4; stage++) { + for (size_t i = 0; i < latency_track_store_count_; i++) { + latencies_[stage][i] = 0; + counts_[stage][i] = 0; + } + } + } + + FawnDS_Combi::~FawnDS_Combi() + { + if (open_) + Close(); + } + + FawnDS_Return + FawnDS_Combi::Create() + { + if (open_) + return ERROR; + + id_ = config_->GetStringValue("child::id"); + + if (config_->ExistsNode("child::key-len") == 0) + key_len_ = atoi(config_->GetStringValue("child::key-len").c_str()); + else + key_len_ = 0; + + if (config_->ExistsNode("child::data-len") == 0) + data_len_ = atoi(config_->GetStringValue("child::data-len").c_str()); + else + data_len_ = 0; + + if (config_->ExistsNode("child::temp-file") == 0) + temp_file_ = config_->GetStringValue("child::temp-file"); + else + temp_file_ = "/tmp"; + + if (config_->ExistsNode("child::stage-limit") == 0) + stage_limit_ = atoi(config_->GetStringValue("child::stage-limit").c_str()); + else + stage_limit_ = 2; + + if (config_->ExistsNode("child::store0-high-watermark") == 0) + store0_high_watermark_ = atoi(config_->GetStringValue("child::store0-high-watermark").c_str()); + else + store0_high_watermark_ = 2; + + if (config_->ExistsNode("child::store0-low-watermark") == 0) + store0_low_watermark_ = atoi(config_->GetStringValue("child::store0-low-watermark").c_str()); + else + store0_low_watermark_ = 1; + if (store0_low_watermark_ < 1 || store0_high_watermark_ <= store0_low_watermark_) + return ERROR; + + if (config_->ExistsNode("child::store1-high-watermark") == 0) + store1_high_watermark_ = atoi(config_->GetStringValue("child::store1-high-watermark").c_str()); + else + store1_high_watermark_ = 1; + + if (config_->ExistsNode("child::store1-low-watermark") == 0) + store1_low_watermark_ = atoi(config_->GetStringValue("child::store1-low-watermark").c_str()); + else + store1_low_watermark_ = 0; + if (store1_high_watermark_ <= store1_low_watermark_) + return ERROR; + + next_ids_.push_back(0); + next_ids_.push_back(0); + next_ids_.push_back(0); + + all_stores_.push_back(std::vector()); + all_stores_.push_back(std::vector()); + all_stores_.push_back(std::vector()); + + all_stores_[0].push_back(alloc_store(0)); + all_stores_[0].back()->Create(); + + back_store_size_ = 0; + + convert_task_running_ = false; + merge_task_running_ = false; + + open_ = true; + + return OK; + } + + FawnDS_Return + FawnDS_Combi::Open() + { + if (open_) + return ERROR; + + // TODO: store and load next IDs for persistency + + id_ = config_->GetStringValue("child::id"); + + if (config_->ExistsNode("child::key-len") == 0) + key_len_ = atoi(config_->GetStringValue("child::key-len").c_str()); + else + key_len_ = 0; + + if (config_->ExistsNode("child::data-len") == 0) + data_len_ = atoi(config_->GetStringValue("child::data-len").c_str()); + else + data_len_ = 0; + + if (config_->ExistsNode("child::temp-file") == 0) + temp_file_ = config_->GetStringValue("child::temp-file"); + else + temp_file_ = "/tmp"; + + if (config_->ExistsNode("child::stage-limit") == 0) + stage_limit_ = atoi(config_->GetStringValue("child::stage-limit").c_str()); + else + stage_limit_ = 2; + + if (config_->ExistsNode("child::store0-high-watermark") == 0) + store0_high_watermark_ = atoi(config_->GetStringValue("child::store0-high-watermark").c_str()); + else + store0_high_watermark_ = 2; + + if (config_->ExistsNode("child::store0-low-watermark") == 0) + store0_low_watermark_ = atoi(config_->GetStringValue("child::store0-low-watermark").c_str()); + else + store0_low_watermark_ = 1; + if (store0_low_watermark_ < 1 || store0_high_watermark_ <= store0_low_watermark_) + return ERROR; + + if (config_->ExistsNode("child::store1-high-watermark") == 0) + store1_high_watermark_ = atoi(config_->GetStringValue("child::store1-high-watermark").c_str()); + else + store1_high_watermark_ = 1; + + if (config_->ExistsNode("child::store1-low-watermark") == 0) + store1_low_watermark_ = atoi(config_->GetStringValue("child::store1-low-watermark").c_str()); + else + store1_low_watermark_ = 0; + if (store1_high_watermark_ <= store1_low_watermark_) + return ERROR; + + next_ids_.push_back(0); + next_ids_.push_back(0); + next_ids_.push_back(0); + + all_stores_.push_back(std::vector()); + all_stores_.push_back(std::vector()); + all_stores_.push_back(std::vector()); + + all_stores_[0].push_back(alloc_store(0)); + all_stores_[0].back()->Create(); + + back_store_size_ = 0; + + convert_task_running_ = false; + merge_task_running_ = false; + + open_ = true; + + return OK; + } + + FawnDS_Return + FawnDS_Combi::Flush() + { + if (!open_) + return ERROR; + + { + tbb::queuing_rw_mutex::scoped_lock lock(mutex_, false); + + FawnDS_Return ret = all_stores_[0][0]->Flush(); + if (ret != OK) + return ret; + } + + GlobalLimits::instance().disable(); + + { + tbb::queuing_rw_mutex::scoped_lock lock; + RateLimiter rate_limiter(0, 1, 1, 10000000L); // poll status every 10 ms or more + + // ensure all tasks to complete + // note: Flush() may fall in a livelock state if there are many pending operations + while (true) { + if (lock.try_acquire(mutex_, true)) { + if (!convert_task_running_ && !merge_task_running_) { + if (!( + (stage_limit_ >= 1 && all_stores_[0].size() >= store0_high_watermark_) || + (stage_limit_ >= 2 && all_stores_[1].size() >= store1_high_watermark_) + )) + break; + } + lock.release(); + } + + rate_limiter.remove_tokens(1); + } + } + + GlobalLimits::instance().enable(); + + struct timeval tv; + if (gettimeofday(&tv, NULL)) + assert(false); + + for (size_t stage = 0; stage < 4; stage++) { + for (size_t i = 0; i < latency_track_store_count_; i++) { + if (counts_[stage][i] != 0) { + fprintf(stdout, "%llu.%06llu: (%s) latency: stage: %zu, i: %zu, sum: %llu ns, count: %llu\n", + static_cast(tv.tv_sec), + static_cast(tv.tv_usec), + id_.c_str(), + stage, i, + static_cast(latencies_[stage][i]), + static_cast(counts_[stage][i])); + } + latencies_[stage][i] = 0; + counts_[stage][i] = 0; + } + } + fflush(stdout); + + return OK; + } + + FawnDS_Return + FawnDS_Combi::Close() + { + if (!open_) + return ERROR; + + FawnDS_Return ret = Flush(); + if (ret != OK) + return ret; + + for (size_t stage = 0; stage < all_stores_.size(); stage++) { + for (size_t i = 0; i < all_stores_[stage].size(); i++) { + all_stores_[stage][i]->Close(); + delete all_stores_[stage][i]; + } + } + + all_stores_.clear(); + + open_ = false; + + return OK; + } + + FawnDS_Return + FawnDS_Combi::Destroy() + { + // TODO: implement + + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS_Combi::Status(const FawnDS_StatusType& type, Value& status) const + { + if (!open_) + return ERROR; + + tbb::queuing_rw_mutex::scoped_lock lock(mutex_, false); + + std::ostringstream oss; + switch (type) { + case NUM_DATA: + case NUM_ACTIVE_DATA: + case MEMORY_USE: + case DISK_USE: + { + Value status_part; + oss << '['; + for (size_t stage = 0; stage < all_stores_.size(); stage++) { + if (stage != 0) + oss << ','; + oss << '['; + for (size_t i = 0; i < all_stores_[stage].size(); i++) { + if (i != 0) + oss << ','; + FawnDS_Return ret = all_stores_[stage][i]->Status(type, status_part); + if (ret != OK) + return UNSUPPORTED; + oss << status_part.str(); + } + oss << ']'; + } + oss << ']'; + } + break; + case CAPACITY: + oss << -1; // unlimited + break; + default: + return UNSUPPORTED; + } + status = NewValue(oss.str()); + return OK; + } + + FawnDS_Return + FawnDS_Combi::Put(const ConstValue& key, const ConstValue& data) + { + if (!open_) + return ERROR; + + if (key_len_ != key.size()) + return INVALID_KEY; + + if (data_len_ != data.size()) + return INVALID_DATA; + + FawnDS_Return ret; + while (true) { + tbb::queuing_rw_mutex::scoped_lock lock(mutex_, false); + FawnDS* front_store = all_stores_[0][0]; + + ret = front_store->Put(key, data); + if (ret != INSUFFICIENT_SPACE) + break; + + lock.upgrade_to_writer(); + + if (front_store != all_stores_[0][0]) { + // other thread already made a new front store + continue; + } + + FawnDS* new_store = alloc_store(0); + new_store->Create(); + + all_stores_[0].insert(all_stores_[0].begin(), new_store); + + if (stage_limit_ >= 1 && + !convert_task_running_ && + all_stores_[0].size() >= store0_high_watermark_) { + convert_task_running_ = true; + + ConvertTask* t = new ConvertTask(); + t->fawnds = this; + task_scheduler_convert_.enqueue_task(t); + } + } + + return ret; + } + + FawnDS_Return + FawnDS_Combi::Append(Value& key, const ConstValue& data) + { + if (!open_) + return ERROR; + + if (data_len_ != data.size()) + return INVALID_DATA; + + FawnDS_Return ret; + while (true) { + tbb::queuing_rw_mutex::scoped_lock lock(mutex_, false); + FawnDS* front_store = all_stores_[0][0]; + + ret = front_store->Append(key, data); + if (ret != INSUFFICIENT_SPACE) + break; + + lock.upgrade_to_writer(); + + if (front_store != all_stores_[0][0]) { + // other thread already made a new front store + continue; + } + + FawnDS* new_store = alloc_store(0); + new_store->Create(); + + all_stores_[0].insert(all_stores_[0].begin(), new_store); + + if (stage_limit_ >= 1 && + !convert_task_running_ && + all_stores_[0].size() >= store0_high_watermark_) { + convert_task_running_ = true; + + ConvertTask* t = new ConvertTask(); + t->fawnds = this; + task_scheduler_convert_.enqueue_task(t); + } + } + + return ret; + } + + FawnDS_Return + FawnDS_Combi::Delete(const ConstValue& key) + { + if (key_len_ != key.size()) + return INVALID_KEY; + + tbb::queuing_rw_mutex::scoped_lock lock(mutex_, false); // Delete() does not change the all_stores_ structure + + FawnDS_Return ret = all_stores_[0][0]->Delete(key); + if (ret == OK || ret == KEY_NOT_FOUND) + return OK; + else + return ret; + } + + FawnDS_Return + FawnDS_Combi::Contains(const ConstValue& key) const + { + if (key_len_ != key.size()) + return INVALID_KEY; + + tbb::queuing_rw_mutex::scoped_lock lock(mutex_, false); + + for (size_t stage = 0; stage < all_stores_.size(); stage++) { + for (size_t i = 0; i < all_stores_[stage].size(); i++) { + FawnDS_Return ret = all_stores_[stage][i]->Contains(key); + if (ret != KEY_NOT_FOUND) + return ret; + } + } + + return KEY_NOT_FOUND; + } + + FawnDS_Return + FawnDS_Combi::Length(const ConstValue& key, size_t& len) const + { + if (key_len_ != key.size()) + return INVALID_KEY; + + tbb::queuing_rw_mutex::scoped_lock lock(mutex_, false); + + for (size_t stage = 0; stage < all_stores_.size(); stage++) { + for (size_t i = 0; i < all_stores_[stage].size(); i++) { + FawnDS_Return ret = all_stores_[stage][i]->Length(key, len); + if (ret != KEY_NOT_FOUND) + return ret; + } + } + + return KEY_NOT_FOUND; + } + + FawnDS_Return + FawnDS_Combi::Get(const ConstValue& key, Value& data, size_t offset, size_t len) const + { + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + int64_t last_time = static_cast(ts.tv_sec) * 1000000000Lu + static_cast(ts.tv_nsec); + + if (key_len_ != key.size()) + return INVALID_KEY; + + tbb::queuing_rw_mutex::scoped_lock lock(mutex_, false); + + for (size_t stage = 0; stage < all_stores_.size(); stage++) { + for (size_t i = 0; i < all_stores_[stage].size(); i++) { + FawnDS_Return ret = all_stores_[stage][i]->Get(key, data, offset, len); + if (ret != KEY_NOT_FOUND) { + clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + int64_t current_time = static_cast(ts.tv_sec) * 1000000000Lu + static_cast(ts.tv_nsec); + if (stage < 3 && i < latency_track_store_count_) { + latencies_[stage][i] += current_time - last_time; + ++counts_[stage][i]; + } + + return ret; + } + } + } + + clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + int64_t current_time = static_cast(ts.tv_sec) * 1000000000Lu + static_cast(ts.tv_nsec); + latencies_[3][0] += current_time - last_time; + ++counts_[3][0]; + + return KEY_NOT_FOUND; + } + + FawnDS_ConstIterator + FawnDS_Combi::Enumerate() const + { + IteratorElem* elem = new IteratorElem(this); + elem->current_stage = 0; + elem->current_store = static_cast(-1); + elem->Next(); + return FawnDS_ConstIterator(elem); + } + + FawnDS_Iterator + FawnDS_Combi::Enumerate() + { + IteratorElem* elem = new IteratorElem(this); + elem->current_stage = 0; + elem->current_store = static_cast(-1); + elem->Next(); + return FawnDS_Iterator(elem); + } + + FawnDS* + FawnDS_Combi::alloc_store(size_t stage, size_t size) + { + char buf[1024]; + snprintf(buf, sizeof(buf), "%zu", stage); + + Configuration* config = new Configuration(config_, true); + config->SetContextNode(std::string("child::store") + buf); + + snprintf(buf, sizeof(buf), "%s_%zu", config_->GetStringValue("child::id").c_str(), next_ids_[stage]++); + config->SetStringValue("child::id", buf); + + snprintf(buf, sizeof(buf), "%zu", key_len_); + config->SetStringValue("child::key-len", buf); + + snprintf(buf, sizeof(buf), "%zu", data_len_); + config->SetStringValue("child::data-len", buf); + + if (size != static_cast(-1)) { + snprintf(buf, sizeof(buf), "%zu", size); + config->SetStringValue("child::size", buf); + } + + FawnDS* store = FawnDS_Factory::New(config); + if (store == NULL) { + DPRINTF(2, "failed to allocate new store\n"); + delete config; + } + return store; + } + + void + FawnDS_Combi::ConvertTask::Run() + { + //fprintf(stderr, "FawnDS_Combi::ConvertTask::Run(): converting\n"); + + FawnDS* front_store; + FawnDS* middle_store; + + //fprintf(stderr, "FawnDS_Combi::ConvertTask::Run(): 1\n"); + + // get the front store + { + tbb::queuing_rw_mutex::scoped_lock lock(fawnds->mutex_, false); + + assert(fawnds->all_stores_[0].size() > fawnds->store0_low_watermark_); + + front_store = fawnds->all_stores_[0].back(); + } + + //fprintf(stderr, "FawnDS_Combi::ConvertTask::Run(): 2\n"); + + // convert to the middle store + { + Value status; + size_t num_data = 0; + if (front_store->Status(NUM_DATA, status) == OK) + num_data = atoll(status.str().c_str()); + //size_t expected_IO = (fawnds->key_len_ + fawnds->data_len_) * num_data; + + middle_store = convert(front_store); + } + + //fprintf(stderr, "FawnDS_Combi::ConvertTask::Run(): 3\n"); + + // remove the front store and insert the middle store + { + tbb::queuing_rw_mutex::scoped_lock lock(fawnds->mutex_, true); + + //fprintf(stderr, "FawnDS_Combi::ConvertTask::Run(): 4\n"); + fawnds->all_stores_[0].pop_back(); + fawnds->all_stores_[1].insert(fawnds->all_stores_[1].begin(), middle_store); + + // check if merge is necessary + if (fawnds->stage_limit_ >= 2 && + !fawnds->merge_task_running_ && + fawnds->all_stores_[1].size() >= fawnds->store1_high_watermark_) { + fawnds->merge_task_running_ = true; + + MergeTask* t = new MergeTask(); + t->fawnds = fawnds; + fawnds->task_scheduler_merge_.enqueue_task(t); + } + } + + // destroy the front store + { + front_store->Close(); + front_store->Destroy(); + delete front_store; + } + + { + tbb::queuing_rw_mutex::scoped_lock lock(fawnds->mutex_, true); + + if (fawnds->all_stores_[0].size() > fawnds->store0_low_watermark_) { + // requeue + ConvertTask* t = new ConvertTask(); + t->fawnds = fawnds; + fawnds->task_scheduler_convert_.enqueue_task(t); + } + else { + // no more store to convert + assert(fawnds->convert_task_running_); + fawnds->convert_task_running_ = false; + } + } + + //fprintf(stderr, "FawnDS_Combi::ConvertTask::Run(): converting done\n"); + } + + FawnDS* + FawnDS_Combi::ConvertTask::convert(FawnDS* front_store) + { + { + struct timeval tv; + if (gettimeofday(&tv, NULL)) { + perror("Error while getting the current time"); + } + + Value status; + front_store->Status(NUM_ACTIVE_DATA, status); + + fprintf(stdout, "%llu.%06llu: (%s) conversion started: %s from front store\n", + static_cast(tv.tv_sec), + static_cast(tv.tv_usec), + fawnds->id_.c_str(), + status.str().c_str() + ); + fflush(stdout); + } + + // locking for alloc_store is unnecessary because there is only one thread that modifies the specific value + FawnDS* middle_store = fawnds->alloc_store(1); + if (middle_store->Create() != OK) { + fprintf(stderr, "Error while creating a middle store\n"); + assert(false); + delete middle_store; + return NULL; + } + + FawnDS_Return ret = front_store->ConvertTo(middle_store); + assert(ret == OK); + + { + struct timeval tv; + if (gettimeofday(&tv, NULL)) { + perror("Error while getting the current time"); + } + + Value status; + middle_store->Status(NUM_ACTIVE_DATA, status); + + fprintf(stdout, "%llu.%06llu: (%s) conversion finished: %s entries to middle store\n", + static_cast(tv.tv_sec), + static_cast(tv.tv_usec), + fawnds->id_.c_str(), + status.str().c_str() + ); + fflush(stdout); + } + + return middle_store; + } + + void + FawnDS_Combi::MergeTask::Run() + { + //fprintf(stderr, "FawnDS_Combi::MergeTask::Run(): merging\n"); + + // check if there is an enough number of middle stores to merge + { + tbb::queuing_rw_mutex::scoped_lock lock(fawnds->mutex_, false); + + assert(fawnds->all_stores_[1].size() > fawnds->store1_low_watermark_); + } + + { + struct timeval tv; + if (gettimeofday(&tv, NULL)) { + perror("Error while getting the current time"); + } + fprintf(stdout, "%llu.%06llu: (%s) merge started\n", + static_cast(tv.tv_sec), + static_cast(tv.tv_usec), + fawnds->id_.c_str() + ); + fflush(stdout); + } + + size_t num_adds = 0; + size_t num_dels = 0; + + // sort all available middle stores + FawnDS* sorter = NULL; + size_t sorted_middle_stores = 0; + while (true) { + FawnDS* middle_store_to_sort; + { + tbb::queuing_rw_mutex::scoped_lock lock(fawnds->mutex_, false); + if (sorted_middle_stores + fawnds->store1_low_watermark_ >= fawnds->all_stores_[1].size()) { + // no more middle store to sort + break; + } + size_t idx = fawnds->all_stores_[1].size() - 1 - sorted_middle_stores; + middle_store_to_sort = fawnds->all_stores_[1][idx]; + } + sorted_middle_stores++; + + sorter = sort(sorter, middle_store_to_sort, num_adds, num_dels); + } + + // obtain the back store + FawnDS* back_store; + { + // don't need to obtain read lock because the current thread is the only thread that can modify fawnds->all_stores_[2] + if (fawnds->all_stores_[2].size() == 0) + back_store = NULL; + else if (fawnds->all_stores_[2].size() == 1) + back_store = fawnds->all_stores_[2][0]; + else { + assert(false); + return; + } + } + + // merge the sorted middle stores into the back store + FawnDS* new_back_store; + { + new_back_store = merge(back_store, sorter, num_adds, num_dels); + } + + // remove the middle stores and replace the back store + std::vector removed_middle_stores; + { + tbb::queuing_rw_mutex::scoped_lock lock(fawnds->mutex_, true); + + for (size_t i = 0; i < sorted_middle_stores; i++) { + removed_middle_stores.push_back(fawnds->all_stores_[1].back()); + fawnds->all_stores_[1].pop_back(); + } + + fawnds->all_stores_[2].clear(); + fawnds->all_stores_[2].push_back(new_back_store); + } + + // destroy middle stores and the back store + { + for (size_t i = 0; i < sorted_middle_stores; i++) { + removed_middle_stores[i]->Close(); + removed_middle_stores[i]->Destroy(); + delete removed_middle_stores[i]; + } + removed_middle_stores.clear(); + + if (back_store) { + back_store->Close(); + back_store->Destroy(); + delete back_store; + back_store = NULL; + } + } + + { + struct timeval tv; + if (gettimeofday(&tv, NULL)) { + perror("Error while getting the current time"); + } + fprintf(stdout, "%llu.%06llu: (%s) merge finished: %zu entries to back store, %zu entries deleted\n", + static_cast(tv.tv_sec), + static_cast(tv.tv_usec), + fawnds->id_.c_str(), + num_adds, + num_dels + ); + fflush(stdout); + } + + { + tbb::queuing_rw_mutex::scoped_lock lock(fawnds->mutex_, true); + if (fawnds->all_stores_[1].size() > fawnds->store1_low_watermark_) { + // requeue + MergeTask* t = new MergeTask(); + t->fawnds = fawnds; + fawnds->task_scheduler_merge_.enqueue_task(t); + } + else { + // no more store to merge + assert(fawnds->merge_task_running_); + fawnds->merge_task_running_ = false; + } + } + + //fprintf(stderr, "FawnDS_Combi::MergeTask::Run(): merging done\n"); + } + + FawnDS* + FawnDS_Combi::MergeTask::sort(FawnDS* sorter, FawnDS* middle_store, size_t& num_adds, size_t& num_dels) + { + if (sorter == NULL) { + Configuration* sorter_config = new Configuration(); + + char buf[1024]; + + if (sorter_config->CreateNodeAndAppend("type", ".") != 0) + assert(false); + if (sorter_config->SetStringValue("type", "sorter") != 0) + assert(false); + + if (sorter_config->CreateNodeAndAppend("key-len", ".") != 0) + assert(false); + snprintf(buf, sizeof(buf), "%zu", fawnds->key_len_); + if (sorter_config->SetStringValue("key-len", buf) != 0) + assert(false); + + if (sorter_config->CreateNodeAndAppend("data-len", ".") != 0) + assert(false); + snprintf(buf, sizeof(buf), "%zu", 1 + fawnds->data_len_); + if (sorter_config->SetStringValue("data-len", buf) != 0) + assert(false); + + if (sorter_config->CreateNodeAndAppend("temp-file", ".") != 0) + assert(false); + if (sorter_config->SetStringValue("temp-file", fawnds->temp_file_) != 0) + assert(false); + + sorter = FawnDS_Factory::New(sorter_config); + if (!sorter) { + assert(false); + delete sorter_config; + return NULL; + } + if (sorter->Create() != OK) { + assert(false); + delete sorter; + return NULL; + } + } + + FawnDS_ConstIterator it = middle_store->Enumerate(); + while (!it.IsEnd()) { + Value combined_data; + combined_data.resize(1 + fawnds->data_len_); + combined_data.data()[0] = it.IsKeyDeleted() ? 1 : 0; + memcpy(combined_data.data() + 1, it->data.data(), it->data.size()); + + if (sorter->Put(it->key, combined_data) != OK) { + assert(false); + delete sorter; + return NULL; + } + + if (!it.IsKeyDeleted()) + num_adds++; + else + num_dels++; + + GlobalLimits::instance().remove_merge_tokens(1); + ++it; + } + + { + struct timeval tv; + if (gettimeofday(&tv, NULL)) { + perror("Error while getting the current time"); + } + fprintf(stdout, "%llu.%06llu: (%s) sorting: added more unsorted entries (currently %zu entries to add, %zu entries to delete)\n", + static_cast(tv.tv_sec), + static_cast(tv.tv_usec), + fawnds->id_.c_str(), + num_adds, + num_dels + ); + fflush(stdout); + } + + return sorter; + } + + FawnDS* + FawnDS_Combi::MergeTask::merge(FawnDS* back_store, FawnDS* sorter, size_t& num_adds, size_t& num_dels) + { + DPRINTF(2, "FawnDS_Combi::MergeTask::Merge(): sorting middle store entries\n"); + + //fprintf(stderr, "FawnDS_Combi::MergeTask::Merge(): %zu entries found in middle stores\n", num_adds + num_dels); + + { + struct timeval tv; + if (gettimeofday(&tv, NULL)) { + perror("Error while getting the current time"); + } + fprintf(stdout, "%llu.%06llu: (%s) sorting: stopped adding unsorted entries\n", + static_cast(tv.tv_sec), + static_cast(tv.tv_usec), + fawnds->id_.c_str() + ); + fflush(stdout); + } + + if (sorter->Flush() != OK) { + assert(false); + delete sorter; + return NULL; + } + + DPRINTF(2, "FawnDS_Combi::MergeTask::Merge(): merging sorted entries into the back store with duplicate entry supression\n"); + + // note that this is just a guess, not an accurate estimate. + // the actual size can vary if there are many updates or duplicate deletions. + ssize_t estimated_new_back_store_size = static_cast(fawnds->back_store_size_) + static_cast(num_adds) - static_cast(num_dels); + size_t max_new_back_store_size; + if (estimated_new_back_store_size <= 0) + max_new_back_store_size = 0; + else + max_new_back_store_size = estimated_new_back_store_size; + + // locking for alloc_store is unnecessary because there is only one thread that modifies the specific value + FawnDS* new_back_store = fawnds->alloc_store(2, max_new_back_store_size); + if (new_back_store->Create() != OK) { + fprintf(stderr, "Error while creating a back store\n"); + assert(false); + delete back_store; + delete sorter; + return NULL; + } + + num_adds = 0; + num_dels = 0; + + bool first = true; + + { + FawnDS_ConstIterator it_b; + if (back_store) + it_b = back_store->Enumerate(); + FawnDS_ConstIterator it_m = sorter->Enumerate(); + + if (first) { + first = false; + + { + struct timeval tv; + if (gettimeofday(&tv, NULL)) { + perror("Error while getting the current time"); + } + fprintf(stdout, "%llu.%06llu: (%s) sorting: started retrieving sorted entries\n", + static_cast(tv.tv_sec), + static_cast(tv.tv_usec), + fawnds->id_.c_str() + ); + fflush(stdout); + } + } + + while (true) { + bool select_middle_store; + + if (!it_m.IsEnd()) { + if (!it_b.IsEnd()) { + int cmp = it_m->key.compare(it_b->key); + if (cmp < 0) + select_middle_store = true; + else if (cmp == 0) { + select_middle_store = true; + // ignore duplicate key from the back store + // by taking the key fro the middle store only + GlobalLimits::instance().remove_merge_tokens(1); + ++it_b; + num_dels++; + } + else + select_middle_store = false; + } + else + select_middle_store = true; + } + else { + if (!it_b.IsEnd()) + select_middle_store = false; + else + break; + } + + Value key; + Value data; + bool deleted; + + if (select_middle_store) { + while (true) { + key = it_m->key; + deleted = it_m->data.data()[0] == 1; + if (!deleted) + data = NewValue(it_m->data.data() + 1, fawnds->data_len_); + else + data.resize(0); + + //GlobalLimits::instance().remove_merge_tokens(1); + // this data is from sorter, which uses another storage, + // so it should not be rate-limited + ++it_m; + + // ignore duplicate key from the middle store + // by taking the last key fro the middle store only + if (!it_m.IsEnd() && key == it_m->key) { + //fprintf(stderr, "dup key: "); + //print_payload((const u_char*)key.data(), key.size(), 0); + //if (data == it_m->data) + // fprintf(stderr, "dup data!\n"); + //else + // fprintf(stderr, "non-dup data!\n"); + num_dels++; + continue; + } + else + break; + } + } + else { + // the back store is assumed to have no duplicate entry + key = it_b->key; + data = it_b->data; + deleted = false; + GlobalLimits::instance().remove_merge_tokens(1); + ++it_b; + } + + if (!deleted) { + FawnDS_Return ret = new_back_store->Put(key, data); + assert(ret == OK); + num_adds++; + } + else + num_dels++; + } + } + + delete sorter; + sorter = NULL; + + //fprintf(stderr, "FawnDS_Combi::MergeTask::Merge(): %zu keys kept, %zu keys deleted\n", num_adds, num_dels); + + DPRINTF(2, "FawnDS_Combi::MergeTask::Merge(): flushing\n"); + + new_back_store->Flush(); + + fawnds->back_store_size_ = num_adds; + + return new_back_store; + } + + FawnDS_Combi::IteratorElem::IteratorElem(const FawnDS_Combi* fawnds) + { + this->fawnds = fawnds; + lock = new tbb::queuing_rw_mutex::scoped_lock(fawnds->mutex_, false); + } + + FawnDS_Combi::IteratorElem::~IteratorElem() + { + delete lock; + lock = NULL; + } + + FawnDS_IteratorElem* + FawnDS_Combi::IteratorElem::Clone() const + { + IteratorElem* elem = new IteratorElem(static_cast(fawnds)); + tbb::queuing_rw_mutex::scoped_lock* elem_lock = elem->lock; + *elem = *this; + elem->lock = elem_lock; + return elem; + } + + void + FawnDS_Combi::IteratorElem::Next() + { + const FawnDS_Combi* fawnds_combi = static_cast(fawnds); + + if (!store_it.IsEnd()) { + ++store_it; + } + + while (store_it.IsEnd()) { + current_store++; + + while (true) { + if (current_stage >= fawnds_combi->all_stores_.size()) { + state = END; + return; + } + + if (current_store >= fawnds_combi->all_stores_[current_stage].size()) { + current_stage++; + fprintf(stderr, "FawnDS_Combi::IteratorElem::Next(): enumerating stage %zu\n", current_stage); + current_store = 0; + } + else + break; + } + + store_it = fawnds_combi->all_stores_[current_stage][current_store]->Enumerate(); + } + + state = store_it->state; + key = store_it->key; + data = store_it->data; + } + + TaskScheduler FawnDS_Combi::task_scheduler_convert_(1, 100, TaskScheduler::CPU_LOW); + TaskScheduler FawnDS_Combi::task_scheduler_merge_(1, 100, TaskScheduler::CPU_LOW); + +} // namespace fawn + diff --git a/fawnds/fawnds_combi.h b/fawnds/fawnds_combi.h new file mode 100644 index 0000000..7103847 --- /dev/null +++ b/fawnds/fawnds_combi.h @@ -0,0 +1,140 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _FAWNDS_COMBI_H_ +#define _FAWNDS_COMBI_H_ + +#include "fawnds.h" +#include "task.h" +#include + +namespace fawn { + + // configuration + // : "combi" (fixed) + // : the ID of the store + // : the file name prefix to store bookkeeping information for persistency + // : the length of keys -- zero for variable-length keys (default), a positive integer for fixed-length keys + // : the length of data -- zero for variable-length data (default), a positive integer for fixed-length data + // : the path of the temporary files/directory. "/tmp" is the default. + // : limit the use of stages up to the specific number (e.g. 1: no store2 will be used); will still need to specify the configuration for all stores. 2 for no limit (default) + // : the number of front stores to begin conversion to a middle store. 2 is default. + // : the number of front stores to stop conversion to a middle store. 1 is default. Must be at least 1 due to the writable store. store. store. store. + // : the number of middle stores to begin merge into the back store. 1 is default. + // : the number of middle stores to stop merge into the back store. 0 is default. + // : the configuration for front stores + // : will be assigned by FawnDS_Combi + // : will be set by FawnDS_Combi + // : will be set by FawnDS_Combi + // : the configuration for middle stores + // : will be assigned by FawnDS_Combi + // : will be set by FawnDS_Combi + // : will be set by FawnDS_Combi + // : the configuration for back stores + // : will be assigned by FawnDS_Combi + // : will be set by FawnDS_Combi + // : will be set by FawnDS_Combi + // : will be set by FawnDS_Combi + + class FawnDS_Combi : public FawnDS { + public: + FawnDS_Combi(); + virtual ~FawnDS_Combi(); + + virtual FawnDS_Return Create(); + virtual FawnDS_Return Open(); + + virtual FawnDS_Return Flush(); + virtual FawnDS_Return Close(); + + virtual FawnDS_Return Destroy(); + + virtual FawnDS_Return Status(const FawnDS_StatusType& type, Value& status) const; + + virtual FawnDS_Return Put(const ConstValue& key, const ConstValue& data); + virtual FawnDS_Return Append(Value& key, const ConstValue& data); + + virtual FawnDS_Return Delete(const ConstValue& key); + + virtual FawnDS_Return Contains(const ConstValue& key) const; + virtual FawnDS_Return Length(const ConstValue& key, size_t& len) const; + virtual FawnDS_Return Get(const ConstValue& key, Value& data, size_t offset = 0, size_t len = -1) const; + + virtual FawnDS_ConstIterator Enumerate() const; + virtual FawnDS_Iterator Enumerate(); + + //virtual FawnDS_ConstIterator Find(const ConstValue& key) const; + //virtual FawnDS_Iterator Find(const ConstValue& key); + + struct IteratorElem : public FawnDS_IteratorElem { + IteratorElem(const FawnDS_Combi* fawnds); + ~IteratorElem(); + + FawnDS_IteratorElem* Clone() const; + void Next(); + + tbb::queuing_rw_mutex::scoped_lock* lock; + size_t current_stage; + size_t current_store; + FawnDS_Iterator store_it; + }; + + protected: + FawnDS* alloc_store(size_t stage, size_t size = -1); + + class ConvertTask : public Task { + public: + virtual void Run(); + FawnDS_Combi* fawnds; + + FawnDS* convert(FawnDS* front_store); + }; + + class MergeTask : public Task { + public: + virtual void Run(); + FawnDS_Combi* fawnds; + + FawnDS* sort(FawnDS* sorter, FawnDS* middle_store, size_t& num_adds, size_t& num_dels); + FawnDS* merge(FawnDS* back_store, FawnDS* sorter, size_t& num_adds, size_t& num_dels); + }; + + private: + bool open_; + + std::string id_; + + size_t key_len_; + size_t data_len_; + + std::string temp_file_; + + size_t stage_limit_; + + size_t store0_high_watermark_; + size_t store0_low_watermark_; + + size_t store1_high_watermark_; + size_t store1_low_watermark_; + + mutable tbb::queuing_rw_mutex mutex_; + + std::vector > all_stores_; // protected by mutex_ + std::vector next_ids_; // protected by mutex_ + + size_t back_store_size_; + + tbb::atomic convert_task_running_; // also protected by mutex_ + tbb::atomic merge_task_running_; // also protected by mutex_ + + friend class MergeTask; + + static TaskScheduler task_scheduler_convert_; + static TaskScheduler task_scheduler_merge_; + + static const size_t latency_track_store_count_ = 100; + mutable tbb::atomic latencies_[4][latency_track_store_count_]; + mutable tbb::atomic counts_[4][latency_track_store_count_]; + }; + +} // namespace fawn + +#endif // #ifndef _FAWNDS_COMBI_H_ diff --git a/fawnds/fawnds_factory.cc b/fawnds/fawnds_factory.cc new file mode 100644 index 0000000..703ef28 --- /dev/null +++ b/fawnds/fawnds_factory.cc @@ -0,0 +1,67 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ + +#include "fawnds_factory.h" +#include "hash_table_default.h" +#include "hash_table_cuckoo.h" +#include "file_store.h" +#include "fawnds.h" +#include "fawnds_sf.h" +#include "fawnds_sf_ordered_trie.h" +#include "fawnds_partition.h" +#include "fawnds_combi.h" +#include "sorter.h" +#include "fawnds_proxy.h" +#include "fawnds_monitor.h" +#include "bdb.h" +#include "debug.h" + +namespace fawn { + + FawnDS* + FawnDS_Factory::New(std::string config_file) + { + DPRINTF(2, "FawnDS_Factory::New(): use config file: %s\n", config_file.c_str()); + Configuration* config = new Configuration(config_file); + return FawnDS_Factory::New(config); + } + + FawnDS* + FawnDS_Factory::New(const Configuration* config) + { + std::string type = config->GetStringValue("child::type"); + DPRINTF(2, "FawnDS_Factory::New(): creating a new instance of type=%s\n", type.c_str()); + FawnDS* result = NULL; + + if (type == "file") + result = new FileStore(); + else if (type == "sf") + result = new FawnDS_SF(); + else if (type == "sf_ordered_trie") + result = new FawnDS_SF_Ordered_Trie(); + else if (type == "partition") + result = new FawnDS_Partition(); + else if (type == "combi") + result = new FawnDS_Combi(); + else if (type == "default") + result = new HashTableDefault(); + else if (type == "cuckoo") + result = new HashTableCuckoo(); + else if (type == "sorter") + result = new Sorter(); + else if (type == "proxy") + result = new FawnDS_Proxy(); + else if (type == "monitor") + result = new FawnDS_Monitor(); +#ifdef HAVE_LIBDB + else if (type == "bdb") + result = new BDB(); +#endif + + if (!result) + return NULL; + + result->SetConfig(config); + return result; + } + +} // namespace fawn diff --git a/fawnds/fawnds_factory.h b/fawnds/fawnds_factory.h new file mode 100644 index 0000000..859ae33 --- /dev/null +++ b/fawnds/fawnds_factory.h @@ -0,0 +1,23 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _FAWNDS_FACTORY_H_ +#define _FAWNDS_FACTORY_H_ + +//using fawn::DataHeader; +//using fawn::HashUtil; + +//using namespace std; + +#include "configuration.h" +#include "fawnds.h" + +namespace fawn { + + class FawnDS_Factory { + public: + static FawnDS* New(std::string config_file); + static FawnDS* New(const Configuration* config); + }; + +} // namespace fawn + +#endif // #ifndef _FAWNDS_FACTORY_H_ diff --git a/fawnds/fawnds_iterator.cc b/fawnds/fawnds_iterator.cc new file mode 100644 index 0000000..6f38931 --- /dev/null +++ b/fawnds/fawnds_iterator.cc @@ -0,0 +1,282 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ + +#include "fawnds_iterator.h" +#include "fawnds.h" +#include "debug.h" + +#include + +namespace fawn { + class FawnDS; + class FawnDSIter; + + FawnDS_IteratorElem::FawnDS_IteratorElem() + : state(END), fawnds(NULL) + { + } + + FawnDS_IteratorElem::~FawnDS_IteratorElem() + { + } + + FawnDS_IteratorElem* + FawnDS_IteratorElem::Clone() const + { + return NULL; + } + + void + FawnDS_IteratorElem::Prev() + { + state = UNSUPPORTED; + } + + void + FawnDS_IteratorElem::Next() + { + state = UNSUPPORTED; + } + + FawnDS_Return + FawnDS_IteratorElem::Replace(const ConstValue& data) + { + (void)data; + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS_IteratorElem::Replace(const ConstValue& key, const ConstValue& data) + { + (void)key; (void)data; + return UNSUPPORTED; + } + + FawnDS_Return + FawnDS_IteratorElem::Delete() + { + return UNSUPPORTED; + } + + FawnDS_ConstIterator::FawnDS_ConstIterator() + : elem(NULL) + { + } + + FawnDS_ConstIterator::FawnDS_ConstIterator(FawnDS_IteratorElem* elem) + : elem(elem) + { + assert(elem); + } + + FawnDS_ConstIterator::FawnDS_ConstIterator(const FawnDS_ConstIterator& it) + : elem(NULL) + { + *this = it; + } + + FawnDS_ConstIterator::~FawnDS_ConstIterator() + { + delete elem; + elem = NULL; + } + + FawnDS_ConstIterator& + FawnDS_ConstIterator::operator=(const FawnDS_ConstIterator& rhs) + { + delete elem; + if (rhs.elem) + elem = rhs.elem->Clone(); + else + elem = NULL; + return *this; + } + + bool + FawnDS_ConstIterator::IsOK() const + { + return elem && elem->state == OK; + } + + bool + FawnDS_ConstIterator::IsError() const + { + return elem && elem->state == ERROR; + } + + bool + FawnDS_ConstIterator::IsUnsupported() const + { + return elem && elem->state == UNSUPPORTED; + } + + bool + FawnDS_ConstIterator::IsKeyDeleted() const + { + return elem && elem->state == KEY_DELETED; + } + + bool + FawnDS_ConstIterator::IsKeyNotFound() const + { + return elem && elem->state == KEY_NOT_FOUND; + } + + bool + FawnDS_ConstIterator::IsInvalidKey() const + { + return elem && elem->state == INVALID_KEY; + } + + bool + FawnDS_ConstIterator::IsInvalidData() const + { + return elem && elem->state == INVALID_DATA; + } + + bool + FawnDS_ConstIterator::IsInvalidLength() const + { + return elem && elem->state == INVALID_LENGTH; + } + + /* + bool + FawnDS_ConstIterator::IsUnused() const + { + return elem && elem->state == UNUSED; + } + */ + + bool + FawnDS_ConstIterator::IsEnd() const + { + return !elem || elem->state == END; + } + + FawnDS_ConstIterator& + FawnDS_ConstIterator::operator--() + { + assert(elem); + elem->Prev(); + return *this; + } + + FawnDS_ConstIterator + FawnDS_ConstIterator::operator--(int) + { + FawnDS_ConstIterator tmp(*this); + operator--(); + return tmp; + } + + FawnDS_ConstIterator& + FawnDS_ConstIterator::operator++() + { + assert(elem); + elem->Next(); + return *this; + } + + FawnDS_ConstIterator + FawnDS_ConstIterator::operator++(int) + { + FawnDS_ConstIterator tmp(*this); + operator++(); + return tmp; + } + + const FawnDS_IteratorElem& + FawnDS_ConstIterator::operator*() const + { + assert(elem); + return *elem; + } + + const FawnDS_IteratorElem* + FawnDS_ConstIterator::operator->() const + { + assert(elem); + return elem; + } + + FawnDS_Iterator::FawnDS_Iterator() + : FawnDS_ConstIterator() + { + } + + FawnDS_Iterator::FawnDS_Iterator(FawnDS_IteratorElem* elem) + : FawnDS_ConstIterator(elem) + { + } + + FawnDS_Iterator::FawnDS_Iterator(const FawnDS_Iterator& it) + : FawnDS_ConstIterator() + { + *this = it; + } + + FawnDS_Iterator& + FawnDS_Iterator::operator=(const FawnDS_Iterator& rhs) + { + return (*this = static_cast(rhs)); + } + + FawnDS_Iterator& + FawnDS_Iterator::operator=(const FawnDS_ConstIterator& rhs) + { + // same as FawnDS_ConstIterator::operator=() + delete elem; + if (rhs.elem) + elem = rhs.elem->Clone(); + else + elem = NULL; + return *this; + } + + FawnDS_Iterator& + FawnDS_Iterator::operator--() + { + assert(elem); + elem->Prev(); + return *this; + } + + FawnDS_Iterator + FawnDS_Iterator::operator--(int) + { + FawnDS_Iterator tmp(*this); + operator--(); + return tmp; + } + + FawnDS_Iterator& + FawnDS_Iterator::operator++() + { + assert(elem); + elem->Next(); + return *this; + } + + FawnDS_Iterator + FawnDS_Iterator::operator++(int) + { + FawnDS_Iterator tmp(*this); + operator++(); + return tmp; + } + + FawnDS_IteratorElem& + FawnDS_Iterator::operator*() + { + assert(elem); + return *elem; + } + + FawnDS_IteratorElem* + FawnDS_Iterator::operator->() + { + assert(elem); + return elem; + } + +} // namespace fawn diff --git a/fawnds/fawnds_iterator.h b/fawnds/fawnds_iterator.h new file mode 100644 index 0000000..4cefa2e --- /dev/null +++ b/fawnds/fawnds_iterator.h @@ -0,0 +1,86 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _FAWNDS_ITERATOR_H_ +#define _FAWNDS_ITERATOR_H_ + +#include "basic_types.h" +#include "fawnds_types.h" +#include "value.h" +#include + +namespace fawn { + + class FawnDS; + + struct FawnDS_IteratorElem { + FawnDS_Return state; + + Value key; + Value data; + + const FawnDS* fawnds; + + FawnDS_IteratorElem(); + virtual ~FawnDS_IteratorElem(); + + virtual FawnDS_IteratorElem* Clone() const; + + virtual void Prev(); + virtual void Next(); + + // the following methods may const_cast(fawnds) + // because these are exposed as non-const to the user only by FawnDS_Iterator + virtual FawnDS_Return Replace(const ConstValue& data); + virtual FawnDS_Return Replace(const ConstValue& key, const ConstValue& data); + virtual FawnDS_Return Delete(); + }; + + struct FawnDS_ConstIterator : public std::iterator { + FawnDS_IteratorElem* elem; + + FawnDS_ConstIterator(); + ~FawnDS_ConstIterator(); + + FawnDS_ConstIterator(FawnDS_IteratorElem* elem); // move + FawnDS_ConstIterator(const FawnDS_ConstIterator& rhs); + FawnDS_ConstIterator& operator=(const FawnDS_ConstIterator& rhs); + + bool IsOK() const; + bool IsError() const; + bool IsUnsupported() const; + bool IsKeyDeleted() const; + bool IsKeyNotFound() const; + bool IsInvalidKey() const; + bool IsInvalidData() const; + bool IsInvalidLength() const; + //bool IsUnused() const; + bool IsEnd() const; + + FawnDS_ConstIterator& operator--(); + FawnDS_ConstIterator operator--(int); + FawnDS_ConstIterator& operator++(); + FawnDS_ConstIterator operator++(int); + + const FawnDS_IteratorElem& operator*() const; + const FawnDS_IteratorElem* operator->() const; + }; + + struct FawnDS_Iterator : public FawnDS_ConstIterator { + FawnDS_Iterator(); + + FawnDS_Iterator(FawnDS_IteratorElem* elem); // move + FawnDS_Iterator(const FawnDS_Iterator& rhs); + FawnDS_Iterator& operator=(const FawnDS_Iterator& rhs); + FawnDS_Iterator& operator=(const FawnDS_ConstIterator& rhs); + + FawnDS_Iterator& operator--(); + FawnDS_Iterator operator--(int); + FawnDS_Iterator& operator++(); + FawnDS_Iterator operator++(int); + + FawnDS_IteratorElem& operator*(); + FawnDS_IteratorElem* operator->(); + }; + +} // namespace fawn + +#endif // #ifndef _FAWNDS_ITERATOR_H_ diff --git a/fawnds/fawnds_monitor.cc b/fawnds/fawnds_monitor.cc new file mode 100644 index 0000000..6c1c103 --- /dev/null +++ b/fawnds/fawnds_monitor.cc @@ -0,0 +1,188 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "fawnds_monitor.h" +#include +#include +#ifndef __APPLE__ +#include +#endif + +namespace fawn { + + FawnDS_Monitor::FawnDS_Monitor() + : exiting_(false), + rate_limiter_(1, 1, 1, 1000000000L) // 1s report interval + { + write_ops_ = 0; + read_ops_ = 0; + } + + FawnDS_Monitor::~FawnDS_Monitor() + { + // dup -- FawnDS_Proxy calls its Close() in destructor + if (store_) + Close(); + } + + FawnDS_Return + FawnDS_Monitor::Create() + { + FawnDS_Return ret = this->FawnDS_Proxy::Create(); + if (store_) { + struct timeval tv; + if (gettimeofday(&tv, NULL)) { + perror("Error while getting the current time"); + } + last_time_ = static_cast(tv.tv_sec) * 1000000000Lu + static_cast(tv.tv_usec) * 1000Lu; + + if (pthread_create(&tid_, NULL, thread_main, this)) { + perror("Error while creating a monitor thread"); + } + } + return ret; + } + + FawnDS_Return + FawnDS_Monitor::Open() + { + FawnDS_Return ret = this->FawnDS_Proxy::Open(); + if (store_) { + struct timeval tv; + if (gettimeofday(&tv, NULL)) { + perror("Error while getting the current time"); + } + last_time_ = static_cast(tv.tv_sec) * 1000000000Lu + static_cast(tv.tv_usec) * 1000Lu; + + if (pthread_create(&tid_, NULL, thread_main, this)) { + perror("Error while creating a monitor thread"); + } + } + return ret; + } + + FawnDS_Return + FawnDS_Monitor::Close() + { + if (store_) { + // this has a race condition issue but should be find if usually only one thread calls Close() + exiting_ = true; + //if (pthread_cancel(tid_)) { + // perror("Error while canceling the monitor thread"); + //} + if (pthread_join(tid_, NULL)) { + perror("Error while stopping the monitor thread"); + } + } + + return this->FawnDS_Proxy::Close(); + } + + + FawnDS_Return + FawnDS_Monitor::Put(const ConstValue& key, const ConstValue& data) + { + ++write_ops_; + return this->FawnDS_Proxy::Put(key, data); + } + + FawnDS_Return + FawnDS_Monitor::Append(Value& key, const ConstValue& data) + { + ++write_ops_; + return this->FawnDS_Proxy::Append(key, data); + } + + FawnDS_Return + FawnDS_Monitor::Delete(const ConstValue& key) + { + ++write_ops_; + return this->FawnDS_Proxy::Delete(key); + } + + FawnDS_Return + FawnDS_Monitor::Contains(const ConstValue& key) const + { + ++read_ops_; + return this->FawnDS_Proxy::Contains(key); + } + + FawnDS_Return + FawnDS_Monitor::Length(const ConstValue& key, size_t& len) const + { + ++read_ops_; + return this->FawnDS_Proxy::Length(key, len); + } + + FawnDS_Return + FawnDS_Monitor::Get(const ConstValue& key, Value& data, size_t offset, size_t len) const + { + ++read_ops_; + return this->FawnDS_Proxy::Get(key, data, offset, len); + } + + void* + FawnDS_Monitor::thread_main(void* arg) + { + FawnDS_Monitor* monitor = reinterpret_cast(arg); + + while (!monitor->exiting_) { + monitor->rate_limiter_.remove_tokens(1); + + size_t read_ops = monitor->read_ops_; + monitor->read_ops_ -= read_ops; + + size_t write_ops = monitor->write_ops_; + monitor->write_ops_ -= write_ops; + + struct timeval tv; + if (gettimeofday(&tv, NULL)) { + perror("Error while getting the current time"); + } + + uint64_t current_time = static_cast(tv.tv_sec) * 1000000000Lu + static_cast(tv.tv_usec) * 1000Lu; + + double time_diff = static_cast(current_time - monitor->last_time_) / 1000000000.; + monitor->last_time_ = current_time; + + size_t rmem_size = 0; +#ifndef __APPLE__ + FILE* fp = fopen("/proc/self/statm", "r"); + if (fp) { + unsigned long vmem; + unsigned long rmem = 0; + if (fscanf(fp, "%lu %lu", &vmem, &rmem) == 2) { + rmem_size = rmem * getpagesize(); + } + fclose(fp); + } +#endif + + Value status_num_active_data; + monitor->Status(NUM_ACTIVE_DATA, status_num_active_data); + + Value status_memory_use; + monitor->Status(MEMORY_USE, status_memory_use); + + fprintf(stdout, "%llu.%06llu: %9.2lf reads/s, %9.2lf writes/s, %7.2lf MB total\n", + static_cast(tv.tv_sec), + static_cast(tv.tv_usec), + static_cast(read_ops) / time_diff, + static_cast(write_ops) / time_diff, + static_cast(rmem_size) / 1000000. + ); + fprintf(stdout, "%llu.%06llu: NUM_ACTIVE_DATA %s\n", + static_cast(tv.tv_sec), + static_cast(tv.tv_usec), + status_num_active_data.str().c_str() + ); + fprintf(stdout, "%llu.%06llu: MEMORY_USE %s\n", + static_cast(tv.tv_sec), + static_cast(tv.tv_usec), + status_memory_use.str().c_str() + ); + fflush(stdout); + } + + return NULL; + } + +} // namespace fawn diff --git a/fawnds/fawnds_monitor.h b/fawnds/fawnds_monitor.h new file mode 100644 index 0000000..1f6a38d --- /dev/null +++ b/fawnds/fawnds_monitor.h @@ -0,0 +1,53 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _FAWNDS_MONITOR_H_ +#define _FAWNDS_MONITOR_H_ + +#include "fawnds_proxy.h" +#include "rate_limiter.h" +#include +#include + +namespace fawn { + + // configuration + // : "monitor" (fixed) + // <(others)>: used by the main store + + class FawnDS_Monitor : public FawnDS_Proxy { + public: + FawnDS_Monitor(); + virtual ~FawnDS_Monitor(); + + virtual FawnDS_Return Create(); + virtual FawnDS_Return Open(); + + virtual FawnDS_Return Close(); + + virtual FawnDS_Return Put(const ConstValue& key, const ConstValue& data); + virtual FawnDS_Return Append(Value& key, const ConstValue& data); + + virtual FawnDS_Return Delete(const ConstValue& key); + + virtual FawnDS_Return Contains(const ConstValue& key) const; + virtual FawnDS_Return Length(const ConstValue& key, size_t& len) const; + virtual FawnDS_Return Get(const ConstValue& key, Value& data, size_t offset = 0, size_t len = -1) const; + + protected: + static void* thread_main(void* arg); + + private: + pthread_t tid_; + + volatile bool exiting_; + + mutable tbb::atomic write_ops_; + mutable tbb::atomic read_ops_; + + uint64_t last_time_; + + RateLimiter rate_limiter_; + }; + +} // namespace fawn + +#endif // #ifndef _FAWNDS_MONITOR_H_ diff --git a/fawnds/fawnds_partition.cc b/fawnds/fawnds_partition.cc new file mode 100644 index 0000000..8d2d9f1 --- /dev/null +++ b/fawnds/fawnds_partition.cc @@ -0,0 +1,340 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "fawnds_partition.h" +#include "fawnds_factory.h" +#include "debug.h" +#include "bit_access.hpp" +#include + +namespace fawn { + + FawnDS_Partition::FawnDS_Partition() + { + } + + FawnDS_Partition::~FawnDS_Partition() + { + Close(); + } + + FawnDS_Return + FawnDS_Partition::Create() + { + if (stores_.size()) + return ERROR; + + alloc_stores(); + + for (size_t i = 0; i < partitions_; i++) { + if (stores_[i]->Create() != OK) { + return ERROR; + } + } + + return OK; + } + + FawnDS_Return + FawnDS_Partition::Open() + { + if (stores_.size()) + return ERROR; + + alloc_stores(); + + for (size_t i = 0; i < partitions_; i++) { + if (stores_[i]->Open() != OK) { + return ERROR; + } + } + + return OK; + } + + FawnDS_Return + FawnDS_Partition::Flush() + { + if (!stores_.size()) + return ERROR; + + for (std::vector::iterator it = stores_.begin(); it != stores_.end(); ++it) { + FawnDS_Return ret = (*it)->Flush(); + if (ret != OK) + return ret; + } + return OK; + } + + FawnDS_Return + FawnDS_Partition::Close() + { + if (!stores_.size()) + return ERROR; + + for (std::vector::iterator it = stores_.begin(); it != stores_.end(); ++it) { + if (*it) { + FawnDS_Return ret = (*it)->Close(); + if (ret != OK) + return ret; + delete *it; + *it = NULL; + } + } + stores_.clear(); + return OK; + } + + FawnDS_Return + FawnDS_Partition::Destroy() + { + for (size_t i = 0; i < partitions_; i++) { + Configuration* storeConfig = new Configuration(config_, true); + storeConfig->SetContextNode("child::store"); + + char buf[1024]; + snprintf(buf, sizeof(buf), "%s_%zu", config_->GetStringValue("child::id").c_str(), i); + storeConfig->SetStringValue("child::id", buf); + + FawnDS* store = FawnDS_Factory::New(storeConfig); + if (!store) { + delete storeConfig; + return ERROR; + } + store->Destroy(); + delete store; + } + + return OK; + } + + FawnDS_Return + FawnDS_Partition::Status(const FawnDS_StatusType& type, Value& status) const + { + std::ostringstream oss; + switch (type) { + case NUM_DATA: + case NUM_ACTIVE_DATA: + case CAPACITY: + case MEMORY_USE: + case DISK_USE: + { + bool first = true; + Value status_part; + oss << '['; + for (std::vector::const_iterator it = stores_.begin(); it != stores_.end(); ++it) { + if (first) + first = false; + else + oss << ','; + FawnDS_Return ret = (*it)->Status(type, status_part); + if (ret != OK) + return UNSUPPORTED; + oss << status_part.str(); + } + oss << ']'; + } + break; + default: + return UNSUPPORTED; + } + status = NewValue(oss.str()); + return OK; + } + + FawnDS_Return + FawnDS_Partition::Put(const ConstValue& key, const ConstValue& data) + { + if (!stores_.size()) + return ERROR; + + size_t p = get_partition(key); + return stores_[p]->Put(key, data); + } + + FawnDS_Return + FawnDS_Partition::Append(Value& key, const ConstValue& data) + { + if (!stores_.size()) + return ERROR; + + size_t p = get_partition(key); + return stores_[p]->Append(key, data); + } + + FawnDS_Return + FawnDS_Partition::Delete(const ConstValue& key) + { + if (!stores_.size()) + return ERROR; + + size_t p = get_partition(key); + return stores_[p]->Delete(key); + } + + FawnDS_Return + FawnDS_Partition::Contains(const ConstValue& key) const + { + if (!stores_.size()) + return ERROR; + + size_t p = get_partition(key); + return stores_[p]->Contains(key); + } + + FawnDS_Return + FawnDS_Partition::Length(const ConstValue& key, size_t& len) const + { + if (!stores_.size()) + return ERROR; + + size_t p = get_partition(key); + return stores_[p]->Length(key, len); + } + + FawnDS_Return + FawnDS_Partition::Get(const ConstValue& key, Value& data, size_t offset, size_t len) const + { + if (!stores_.size()) + return ERROR; + + size_t p = get_partition(key); + return stores_[p]->Get(key, data, offset, len); + } + + FawnDS_ConstIterator + FawnDS_Partition::Enumerate() const + { + IteratorElem* elem = new IteratorElem(this); + elem->next_store = 0; + elem->Next(); + return FawnDS_ConstIterator(elem); + } + + FawnDS_Iterator + FawnDS_Partition::Enumerate() + { + IteratorElem* elem = new IteratorElem(this); + elem->next_store = 0; + elem->Next(); + return FawnDS_Iterator(elem); + } + + FawnDS_ConstIterator + FawnDS_Partition::Find(const ConstValue& key) const + { + if (!stores_.size()) + return FawnDS_ConstIterator(); + + size_t p = get_partition(key); + return stores_[p]->Find(key); + } + + FawnDS_Iterator + FawnDS_Partition::Find(const ConstValue& key) + { + if (!stores_.size()) + return FawnDS_Iterator(); + + size_t p = get_partition(key); + return stores_[p]->Find(key); + } + + FawnDS_Return + FawnDS_Partition::alloc_stores() + { + skip_bits_ = atoi(config_->GetStringValue("child::skip_bits").c_str()); + + partitions_ = atoi(config_->GetStringValue("child::partitions").c_str()); + partition_bits_ = 0; + while ((1u << partition_bits_) < partitions_) + partition_bits_++; + + if (partitions_ == 0) { + DPRINTF(2, "FawnDS_Partition::alloc_stores(): non-zero partitions required\n"); + return ERROR; + } + + if ((1u << partition_bits_) != partitions_) { + DPRINTF(2, "FawnDS_Partition::alloc_stores(): # of partitions should be power of 2\n"); + return ERROR; + } + + for (size_t i = 0; i < partitions_; i++) { + Configuration* storeConfig = new Configuration(config_, true); + storeConfig->SetContextNode("child::store"); + + char buf[1024]; + snprintf(buf, sizeof(buf), "%s_%zu", config_->GetStringValue("child::id").c_str(), i); + storeConfig->SetStringValue("child::id", buf); + + FawnDS* store = FawnDS_Factory::New(storeConfig); + if (!store) { + delete storeConfig; + return ERROR; + } + stores_.push_back(store); + } + return OK; + } + + size_t + FawnDS_Partition::get_partition(const ConstValue& key) const + { + std::size_t bits = 0; + std::size_t index = 0; + while (bits < partition_bits_) + { + index <<= 1; + if ((skip_bits_ + bits) / 8 > key.size()) + { + // too short key + assert(false); + break; + } + if (cindex::bit_access::get(reinterpret_cast(key.data()), skip_bits_ + bits)) + index |= 1; + bits++; + } + + return index; + } + + FawnDS_Partition::IteratorElem::IteratorElem(const FawnDS_Partition* fawnds) + { + this->fawnds = fawnds; + } + + FawnDS_Partition::IteratorElem::~IteratorElem() + { + } + + FawnDS_IteratorElem* + FawnDS_Partition::IteratorElem::Clone() const + { + IteratorElem* elem = new IteratorElem(static_cast(fawnds)); + *elem = *this; + return elem; + } + + void + FawnDS_Partition::IteratorElem::Next() + { + const FawnDS_Partition* fawnds_part = static_cast(fawnds); + + if (!store_it.IsEnd()) + ++store_it; + + while (store_it.IsEnd()) { + if (next_store >= fawnds_part->stores_.size()) { + state = END; + return; + } + store_it = fawnds_part->stores_[next_store++]->Enumerate(); + } + + state = store_it->state; + key = store_it->key; + data = store_it->data; + } + +} // namespace fawn + diff --git a/fawnds/fawnds_partition.h b/fawnds/fawnds_partition.h new file mode 100644 index 0000000..8628c2f --- /dev/null +++ b/fawnds/fawnds_partition.h @@ -0,0 +1,74 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _FAWNDS_PARTITION_H_ +#define _FAWNDS_PARTITION_H_ + +#include "fawnds.h" +#include + +namespace fawn { + + // configuration + // : "partition" (fixed) + // : the ID of the store + // : the number of MSBs to ignore when calculating the partition number + // : the number of partitions; must be power of 2 + // : the configuration for the lower-level store + // : will be assigned by FawnDS_Partition + + class FawnDS_Partition : public FawnDS { + public: + FawnDS_Partition(); + virtual ~FawnDS_Partition(); + + virtual FawnDS_Return Create(); + virtual FawnDS_Return Open(); + + virtual FawnDS_Return Flush(); + virtual FawnDS_Return Close(); + + virtual FawnDS_Return Destroy(); + + virtual FawnDS_Return Status(const FawnDS_StatusType& type, Value& status) const; + + virtual FawnDS_Return Put(const ConstValue& key, const ConstValue& data); + virtual FawnDS_Return Append(Value& key, const ConstValue& data); + + virtual FawnDS_Return Delete(const ConstValue& key); + + virtual FawnDS_Return Contains(const ConstValue& key) const; + virtual FawnDS_Return Length(const ConstValue& key, size_t& len) const; + virtual FawnDS_Return Get(const ConstValue& key, Value& data, size_t offset = 0, size_t len = -1) const; + + virtual FawnDS_ConstIterator Enumerate() const; + virtual FawnDS_Iterator Enumerate(); + + virtual FawnDS_ConstIterator Find(const ConstValue& key) const; + virtual FawnDS_Iterator Find(const ConstValue& key); + + struct IteratorElem : public FawnDS_IteratorElem { + IteratorElem(const FawnDS_Partition* fawnds); + ~IteratorElem(); + + FawnDS_IteratorElem* Clone() const; + void Next(); + + size_t next_store; + FawnDS_Iterator store_it; + }; + + private: + FawnDS_Return alloc_stores(); + size_t get_partition(const ConstValue& key) const; + + size_t skip_bits_; + size_t partitions_; + size_t partition_bits_; + + std::vector stores_; + + friend class IteratorElem; + }; + +} // namespace fawn + +#endif // #ifndef _FAWNDS_PARTITION_H_ diff --git a/fawnds/fawnds_proxy.cc b/fawnds/fawnds_proxy.cc new file mode 100644 index 0000000..14e036e --- /dev/null +++ b/fawnds/fawnds_proxy.cc @@ -0,0 +1,168 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "fawnds_proxy.h" +#include "fawnds_factory.h" +#include + +namespace fawn { + + FawnDS_Proxy::FawnDS_Proxy() + : store_(NULL) + { + } + + FawnDS_Proxy::~FawnDS_Proxy() + { + if (store_) + Close(); + } + + FawnDS_Return + FawnDS_Proxy::Create() + { + if (store_) + return ERROR; + + Configuration* store_config = new Configuration(config_, true); + if (store_config->DeleteNode("child::type") != 0) + assert(false); + store_ = FawnDS_Factory::New(store_config); + if (!store_) { + delete store_config; + return ERROR; + } + + return store_->Create(); + } + + FawnDS_Return + FawnDS_Proxy::Open() + { + if (store_) + return ERROR; + + Configuration* store_config = new Configuration(config_, true); + if (store_config->DeleteNode("child::type") != 0) + assert(false); + store_ = FawnDS_Factory::New(store_config); + if (!store_) { + delete store_config; + return ERROR; + } + + return store_->Open(); + } + + FawnDS_Return + FawnDS_Proxy::ConvertTo(FawnDS* new_store) const + { + if (!store_) + return ERROR; + + return store_->ConvertTo(new_store); + } + + FawnDS_Return + FawnDS_Proxy::Flush() + { + return store_->Flush(); + } + + FawnDS_Return + FawnDS_Proxy::Close() + { + if (!store_) + return ERROR; + + FawnDS_Return ret = store_->Close(); + if (ret == OK) { + delete store_; + store_ = NULL; + } + return ret; + } + + FawnDS_Return + FawnDS_Proxy::Destroy() + { + Configuration* store_config = new Configuration(config_, true); + if (store_config->DeleteNode("child::type") != 0) + assert(false); + FawnDS* store = FawnDS_Factory::New(store_config); + if (!store_) { + delete store_config; + return ERROR; + } + + FawnDS_Return ret = store->Destroy(); + + delete store; + return ret; + } + + FawnDS_Return + FawnDS_Proxy::Status(const FawnDS_StatusType& type, Value& status) const + { + return store_->Status(type, status); + } + + FawnDS_Return + FawnDS_Proxy::Put(const ConstValue& key, const ConstValue& data) + { + return store_->Put(key, data); + } + + FawnDS_Return + FawnDS_Proxy::Append(Value& key, const ConstValue& data) + { + return store_->Append(key, data); + } + + FawnDS_Return + FawnDS_Proxy::Delete(const ConstValue& key) + { + return store_->Delete(key); + } + + FawnDS_Return + FawnDS_Proxy::Contains(const ConstValue& key) const + { + return store_->Contains(key); + } + + FawnDS_Return + FawnDS_Proxy::Length(const ConstValue& key, size_t& len) const + { + return store_->Length(key, len); + } + + FawnDS_Return + FawnDS_Proxy::Get(const ConstValue& key, Value& data, size_t offset, size_t len) const + { + return store_->Get(key, data, offset, len); + } + + FawnDS_ConstIterator + FawnDS_Proxy::Enumerate() const + { + return store_->Enumerate(); + } + + FawnDS_Iterator + FawnDS_Proxy::Enumerate() + { + return store_->Enumerate(); + } + + FawnDS_ConstIterator + FawnDS_Proxy::Find(const ConstValue& key) const + { + return store_->Find(key); + } + + FawnDS_Iterator + FawnDS_Proxy::Find(const ConstValue& key) + { + return store_->Find(key); + } + +} // namespace fawn diff --git a/fawnds/fawnds_proxy.h b/fawnds/fawnds_proxy.h new file mode 100644 index 0000000..0767de1 --- /dev/null +++ b/fawnds/fawnds_proxy.h @@ -0,0 +1,51 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _FAWNDS_PROXY_H_ +#define _FAWNDS_PROXY_H_ + +#include "fawnds.h" + +namespace fawn { + + // configuration + // : "proxy" (fixed) + // <(others)>: used by the main store + + class FawnDS_Proxy : public FawnDS { + public: + FawnDS_Proxy(); + virtual ~FawnDS_Proxy(); + + virtual FawnDS_Return Create(); + virtual FawnDS_Return Open(); + + virtual FawnDS_Return ConvertTo(FawnDS* new_store) const; + + virtual FawnDS_Return Flush(); + virtual FawnDS_Return Close(); + + virtual FawnDS_Return Destroy(); + + virtual FawnDS_Return Status(const FawnDS_StatusType& type, Value& status) const; + + virtual FawnDS_Return Put(const ConstValue& key, const ConstValue& data); + virtual FawnDS_Return Append(Value& key, const ConstValue& data); + + virtual FawnDS_Return Delete(const ConstValue& key); + + virtual FawnDS_Return Contains(const ConstValue& key) const; + virtual FawnDS_Return Length(const ConstValue& key, size_t& len) const; + virtual FawnDS_Return Get(const ConstValue& key, Value& data, size_t offset = 0, size_t len = -1) const; + + virtual FawnDS_ConstIterator Enumerate() const; + virtual FawnDS_Iterator Enumerate(); + + virtual FawnDS_ConstIterator Find(const ConstValue& key) const; + virtual FawnDS_Iterator Find(const ConstValue& key); + + protected: + FawnDS* store_; + }; + +} // namespace fawn + +#endif // #ifndef _FAWNDS_PROXY_H_ diff --git a/fawnds/fawnds_sf.cc b/fawnds/fawnds_sf.cc new file mode 100644 index 0000000..5bf28e3 --- /dev/null +++ b/fawnds/fawnds_sf.cc @@ -0,0 +1,989 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "fawnds_sf.h" +#include "file_io.h" +#include "configuration.h" +#include "fawnds_factory.h" +#include "print.h" +#include "debug.h" +#include "global_limits.h" +#include +#include + +#include +#include +#include + +namespace fawn { + +/***************************************************/ +/******** DB PUBLIC CREATION/OPEN FUNCTIONS ********/ +/***************************************************/ + +// User must specify what kind of backing to use for the +// hash table: MALLOC_HASHTABLE or MMAP_HASHTABLE. +// When in doubt, use MALLOC_HASHTABLE to avoid any random +// writes to the storage device. + +// MMAP_HASHTABLE might be useful in some circumstances. While +// mmap does flush writes to disk periodically, newer flash +// devices can handle some random writes +// across a *small* address space using block remapping (see +// Polte/Simsa/Gibson WISH2009 paper). One might trade off a bit +// of this performance hit to support more small objects. This +// technique should be approached only after optimizing the +// hashtable structure per-object memory requirements and +// increasing the number of vnodes (reducing the number of objects +// stored per file). + +// As MMAP_HASHTABLE never got used, we removed it. + + + +/***************************************************/ +/***************** DB CONSTRUCTOR ******************/ +/***************************************************/ + +FawnDS_SF::FawnDS_SF() + : header_(NULL), hash_table_(NULL), data_store_(NULL) +{ + pthread_rwlock_init(&fawnds_lock_, NULL); +} + +FawnDS_SF::~FawnDS_SF() +{ + if (header_) + Close(); + pthread_rwlock_destroy(&fawnds_lock_); +} + +FawnDS_Return +FawnDS_SF::Create() +{ + DPRINTF(2, "FawnDS_SF::Create(): creating header\n"); + + if (config_->ExistsNode("child::key-len") == 0) + key_len_ = atoi(config_->GetStringValue("child::key-len").c_str()); + else + key_len_ = 0; + + if (config_->ExistsNode("child::data-len") == 0) + data_len_ = atoi(config_->GetStringValue("child::data-len").c_str()); + else + data_len_ = 0; + + header_ = new DbHeader; + if (header_ == NULL) { + perror("could not create new header file\n"); + return ERROR; + } + // zero out the buffer + memset(header_, 0, sizeof(DbHeader)); + + // populate the database header. + header_->num_elements = 0; + header_->num_active_elements = 0; + //fawnds->header_->max_deleted_ratio = MAX_DELETED_RATIO; + //fawnds->header_->max_load_factor = MAX_LOAD_FACTOR; + + DPRINTF(2, "FawnDS_SF::Create(): creating hash table\n"); + + Configuration* tableConfig = new Configuration(config_, true); + tableConfig->SetContextNode("child::hashtable"); + tableConfig->SetStringValue("child::id", config_->GetStringValue("child::id")); + hash_table_ = FawnDS_Factory::New(tableConfig); + + if (hash_table_->Create() != OK) { + perror("Could not initialize hash_table\n"); + return ERROR; + } + + DPRINTF(2, "FawnDS_SF::Create(): creating data store\n"); + + Configuration* storeConfig = new Configuration(config_, true); + storeConfig->SetContextNode("child::datastore"); + storeConfig->SetStringValue("child::id", config_->GetStringValue("child::id")); + if (key_len_ == 0 || data_len_ == 0) { + storeConfig->SetStringValue("child::data-len", "0"); + } + else { + char buf[1024]; + if (key_len_ == 0) + snprintf(buf, sizeof(buf), "%zu", sizeof(DataHeaderFull) + key_len_ + data_len_); + else + snprintf(buf, sizeof(buf), "%zu", sizeof(DataHeaderSimple) + key_len_ + data_len_); + storeConfig->SetStringValue("child::data-len", buf); + } + data_store_ = FawnDS_Factory::New(storeConfig); + if (data_store_ == NULL) { + DPRINTF(2, "FawnDS_SF::Create(): could not create data store\n"); + return ERROR; + } + + if (data_store_->Create() != OK) + return ERROR; + + DPRINTF(2, "FawnDS_SF::Create(): done\n"); + + return OK; +} + +FawnDS_Return +FawnDS_SF::Open() +{ + if (config_->ExistsNode("child::key-len") == 0) + key_len_ = atoi(config_->GetStringValue("child::key-len").c_str()); + else + key_len_ = 0; + + if (config_->ExistsNode("child::data-len") == 0) + data_len_ = atoi(config_->GetStringValue("child::data-len").c_str()); + else + data_len_ = 0; + + DPRINTF(2, "FawnDS_SF::Open(): reading header\n"); + + if (ReadHeaderFromFile() != 0) { + perror("Could not read header from file\n"); + printf("Populating FawnDS from file failed\n"); + return ERROR; + } + + DPRINTF(2, "FawnDS_SF::Open(): opening hash table\n"); + + Configuration* tableConfig = new Configuration(config_, true); + tableConfig->SetContextNode("child::hashtable"); + tableConfig->SetStringValue("child::id", config_->GetStringValue("child::id")); + hash_table_ = FawnDS_Factory::New(tableConfig); + + if (hash_table_->Open() != OK) { + perror("Could not read hash_table from file\n"); + printf("Populating FawnDS from file failed\n"); + return ERROR; + } + + DPRINTF(2, "FawnDS_SF::Open(): opening data store\n"); + + Configuration* storeConfig = new Configuration(config_, true); + storeConfig->SetContextNode("child::datastore"); + storeConfig->SetStringValue("child::id", config_->GetStringValue("child::id")); + if (key_len_ != 0 && data_len_ != 0) { + char buf[1024]; + snprintf(buf, sizeof(buf), "%zu", key_len_ + data_len_); + storeConfig->SetStringValue("child::data-len", buf); + } + data_store_ = FawnDS_Factory::New(storeConfig); + + if (data_store_->Open() != OK) + return ERROR; + + DPRINTF(2, "FawnDS_SF::Open(): done\n"); + + return OK; +} + +FawnDS_Return +FawnDS_SF::ConvertTo(FawnDS* new_store) const +{ + pthread_rwlock_rdlock(&fawnds_lock_); + + FawnDS_SF* sf = dynamic_cast(new_store); + if (!sf) { + pthread_rwlock_unlock(&fawnds_lock_); + return ERROR; + } + + if (key_len_ != sf->key_len_) { + fprintf(stderr, "FawnDS_SF::ConvertTo(): key length mismatch\n"); + pthread_rwlock_unlock(&fawnds_lock_); + return ERROR; + } + + if (data_len_ != sf->data_len_) { + fprintf(stderr, "FawnDS_SF::ConvertTo(): data length mismatch\n"); + pthread_rwlock_unlock(&fawnds_lock_); + return ERROR; + } + + if (hash_table_->ConvertTo(sf->hash_table_) != OK) { + fprintf(stderr, "FawnDS_SF::ConvertTo(): could not convert hash table\n"); + pthread_rwlock_unlock(&fawnds_lock_); + return ERROR; + } + + //fprintf(stderr, "FawnDS_SF::ConvertTo(): analyzing hash table changes\n"); + + if (sf->config_->ExistsNode("child::keep-datastore-on-conversion") == 0 && + atoi(sf->config_->GetStringValue("child::keep-datastore-on-conversion").c_str()) != 0) { + if (data_store_->ConvertTo(sf->data_store_) != OK) { + fprintf(stderr, "FawnDS_SF::ConvertTo(): could not convert data store\n"); + pthread_rwlock_unlock(&fawnds_lock_); + return ERROR; + } + + sf->header_->num_elements = header_->num_elements; + sf->header_->num_active_elements = header_->num_active_elements; + + pthread_rwlock_unlock(&fawnds_lock_); + return OK; + } + + // side-by-side hash table comparison for data reorganization on data store + + FawnDS_Iterator from_it = hash_table_->Enumerate(); + FawnDS_Iterator to_it = sf->hash_table_->Enumerate(); + + std::vector > mapping; + //std::vector > rev_mapping; + + while (true) { + if (!from_it.IsEnd() && !to_it.IsEnd()) { + uint32_t from = from_it->data.as_number(); + uint32_t to = to_it->data.as_number(); + //fprintf(stderr, "FawnDS_SF::ConvertTo(): mapping %zu => %zu\n", static_cast(from), static_cast(to)); + mapping.push_back(std::make_pair(from, to)); + //rev_mapping.push_back(std::make_pair(to, from)); + } + else if (from_it.IsEnd() && to_it.IsEnd()) + break; + else { + // hash table conversion should have failed + assert(false); + pthread_rwlock_unlock(&fawnds_lock_); + return ERROR; + } + GlobalLimits::instance().remove_convert_tokens(1); + ++from_it; + GlobalLimits::instance().remove_convert_tokens(1); + ++to_it; + } + std::sort(mapping.begin(), mapping.end()); + //std::sort(rev_mapping.begin(), rev_mapping.end()); + + // sequential read, random write (hoping OS adds writeback caching) + std::vector >::const_iterator it; + Value data; + + sf->header_->num_elements = 0; + sf->header_->num_active_elements = 0; + + FawnDS_ConstIterator it_from = data_store_->Enumerate(); + for (it = mapping.begin(); it != mapping.end(); ++it) { + uint32_t from = (*it).first; + uint32_t to = (*it).second; + + while (!it_from.IsEnd() && it_from->key.as_number() < from) + ++it_from; + assert(!it_from.IsEnd() && it_from->key.as_number() == from); + + FawnDS_Return ret = sf->data_store_->Put(RefValue(&to), it_from->data); + if (ret != OK) { + pthread_rwlock_unlock(&fawnds_lock_); + return ERROR; + } + if (sf->header_->num_elements < to + 1) + sf->header_->num_elements = to + 1; + sf->header_->num_active_elements++; + } + + /* + // brute-force reorganization; read all data, write all data. + // TODO: use Nsort for this + std::map read_data; + FawnDS_ConstIterator it_from = data_store_->Enumerate(); + for (it = mapping.begin(); it != mapping.end(); ++it) { + uint32_t from = (*it).first; + uint32_t to = (*it).second; + + while (!it_from.IsEnd() && it_from->key.as_number() < from) + ++it_from; + assert(!it_from.IsEnd() && it_from->key.as_number() == from); + + read_data.insert(std::make_pair(to, NewValue(it_from->data.data(), it_from->data.size()))); + + if (sf->header_->num_elements < to + 1) + sf->header_->num_elements = to + 1; + sf->header_->num_active_elements++; + } + + Value dummy_data; + assert(key_len_ != 0 && data_len_ != 0); + dummy_data.resize(sizeof(DataHeaderSimple) + key_len_ + data_len_, false); + memset(dummy_data.data(), 0, sizeof(DataHeaderSimple)); + memset(dummy_data.data() + sizeof(DataHeaderSimple), 0x55, key_len_ + data_len_); + + std::map::const_iterator it_read_data = read_data.begin(); + Value data_store_key; + for (uint32_t to = 0; to < sf->header_->num_elements; to++) { + FawnDS_Return ret; + if (to < (*it_read_data).first) + ret = sf->data_store_->Append(data_store_key, dummy_data); + else if (to == (*it_read_data).first) { + ret = sf->data_store_->Append(data_store_key, (*it_read_data).second); + ++it_read_data; + } + else + assert(false); + if (ret != OK) { + pthread_rwlock_unlock(&fawnds_lock_); + return ERROR; + } + } + assert(it_read_data == read_data.end()); + */ + + /* + // removes any holes to avoid a sparse file, + // which will decrease direct I/O perfromance + uint32_t last_to = static_cast(-1); + assert(last_to + 1 == 0); + + assert(key_len_ != 0 && data_len_ != 0); + data.resize(sizeof(DataHeaderSimple) + key_len_ + data_len_, false); + memset(data.data(), 0, sizeof(DataHeaderSimple)); + memset(data.data() + sizeof(DataHeaderSimple), 0x55, key_len_ + data_len_); + + for (it = rev_mapping.begin(); it != rev_mapping.end(); ++it) { + uint32_t to = (*it).first; + while (++last_to != to) { + FawnDS_Return ret = sf->data_store_->Put(RefValue(&last_to), data); + if (ret != OK) { + pthread_rwlock_unlock(&fawnds_lock_); + return ERROR; + } + } + } + */ + + //fprintf(stderr, "FawnDS_SF::ConvertTo(): flushing data store\n"); + + FawnDS_Return ret = sf->data_store_->Flush(); + if (ret != OK) { + pthread_rwlock_unlock(&fawnds_lock_); + return ERROR; + } + pthread_rwlock_unlock(&fawnds_lock_); + return OK; +} + + +/***************************************************/ +/****************** DB DESTRUCTOR ******************/ +/***************************************************/ + +FawnDS_Return +FawnDS_SF::Flush() +{ + pthread_rwlock_wrlock(&fawnds_lock_); + bool success = WriteHeaderToFile() == OK && data_store_->Flush() == OK && hash_table_->Flush() == OK; + pthread_rwlock_unlock(&fawnds_lock_); + if (!success) + return ERROR; + return OK; +} + +FawnDS_Return +FawnDS_SF::Close() +{ + if (!header_) + return ERROR; + + DPRINTF(2, "FawnDS_SF::Close(): closing\n"); + + pthread_rwlock_wrlock(&fawnds_lock_); + + delete header_; + header_ = NULL; + delete hash_table_; + hash_table_ = NULL; + delete data_store_; + data_store_ = NULL; + + pthread_rwlock_unlock(&fawnds_lock_); + + return OK; +} + +FawnDS_Return +FawnDS_SF::Destroy() +{ + if (header_) + return ERROR; + + string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + if (unlink(filename.c_str())) { + //fprintf(stderr, "could not delete file: %s: %s\n", filename.c_str(), strerror(errno)); + } + + Configuration* tableConfig = new Configuration(config_, true); + tableConfig->SetContextNode("child::hashtable"); + tableConfig->SetStringValue("child::id", config_->GetStringValue("child::id")); + FawnDS* hash_table = FawnDS_Factory::New(tableConfig); + FawnDS_Return ret_destroy_hash_table = hash_table->Destroy(); + delete hash_table; + + Configuration* storeConfig = new Configuration(config_, true); + storeConfig->SetContextNode("child::datastore"); + storeConfig->SetStringValue("child::id", config_->GetStringValue("child::id")); + FawnDS* data_store = FawnDS_Factory::New(storeConfig); + FawnDS_Return ret_destroy_data_store = data_store->Destroy(); + delete data_store; + + if (ret_destroy_hash_table == OK && ret_destroy_data_store == OK) + return OK; + else + return ERROR; +} + +FawnDS_Return +FawnDS_SF::Status(const FawnDS_StatusType& type, Value& status) const +{ + if (!header_) + return ERROR; + + std::ostringstream oss; + switch (type) { + case NUM_DATA: + oss << header_->num_elements; + break; + case NUM_ACTIVE_DATA: + oss << header_->num_active_elements; + break; + case CAPACITY: + return hash_table_->Status(type, status); + case MEMORY_USE: + case DISK_USE: + { + Value status_hash_table; + Value status_data_store; + if (hash_table_->Status(type, status_hash_table) != OK) + return UNSUPPORTED; + if (data_store_->Status(type, status_data_store) != OK) + return UNSUPPORTED; + oss << '[' << status_hash_table.str(); + oss << ',' << status_data_store.str(); + oss << ']'; + } + break; + default: + return UNSUPPORTED; + } + status = NewValue(oss.str()); + return OK; +} + +/***************************************************/ +/****************** DB FUNCTIONS *******************/ +/***************************************************/ + +FawnDS_Return +FawnDS_SF::Put(const ConstValue& key, const ConstValue& data) +{ +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FawnDS_SF::Put(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + DPRINTF(2, "FawnDS_SF::Put(): data=\n"); + print_payload((const u_char*)data.data(), data.size(), 4); + } +#endif + + if (key.size() == 0) + return INVALID_KEY; + + if (key_len_ != 0 && key_len_ != key.size()) + return INVALID_KEY; + + if (data_len_ != 0 && data_len_ != data.size()) + return INVALID_DATA; + + return InsertEntry(key, data, false); +} + +FawnDS_Return +FawnDS_SF::Delete(const ConstValue& key) +{ +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FawnDS_SF::Delete(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + } +#endif + + if (key.size() == 0) + return INVALID_KEY; + + if (key_len_ != 0 && key_len_ != key.size()) + return INVALID_KEY; + + Value empty_v; + return InsertEntry(key, empty_v, true); +} + +FawnDS_Return +FawnDS_SF::InsertEntry(const ConstValue& key, const ConstValue& data, bool isDelete) +{ + assert(!isDelete || (isDelete && data.size() == 0)); + + pthread_rwlock_wrlock(&fawnds_lock_); + + bool newObj = true; + + FawnDS_Iterator hash_it = hash_table_->Find(key); + size_t dhSize; + if (key_len_ == 0) + dhSize = sizeof(DataHeaderFull); + else + dhSize = sizeof(DataHeaderSimple); + + // Look for an existing entry in the hashtable so that we can update it or invalidate it + SizedValue<64> read_hdr; + Value data_store_key; + while (!hash_it.IsEnd()) { + data_store_key = hash_it->data; + + if (data_store_->Get(data_store_key, read_hdr, 0, dhSize) != OK || + (key_len_ == 0 && read_hdr.as().key_len != key.size())) { + DPRINTF(2, "FawnDS_SF::InsertEntry(): key length mismatch; continue\n"); + ++hash_it; + continue; + } + + DPRINTF(2, "FawnDS_SF::InsertEntry(): key length match\n"); + + SizedValue<64> read_key; + if (data_store_->Get(data_store_key, read_key, dhSize, key.size()) != OK || + key != read_key) { + DPRINTF(2, "FawnDS_SF::InsertEntry(): key mismatch; continue\n"); + ++hash_it; + continue; + } + + DPRINTF(2, "FawnDS_SF::InsertEntry(): key match\n"); + newObj = false; + break; + } + + if (hash_it.IsEnd()) { + DPRINTF(2, "FawnDS_SF::InsertEntry(): key not found in hash table\n"); + } + + if (!newObj) { + // attempt to delete existing object (OK to fail) + data_store_->Delete(key); + } + + Value entry_id; + + // add new entry + + // TODO: can't we use gather output again? + SizedValue<4096> buf; + + buf.resize(dhSize + key.size() + data.size(), false); + if (key_len_ == 0) { + DataHeaderFull dh; + dh.key_len = key.size(); + dh.type = isDelete ? 2 : 1; + memcpy(buf.data(), &dh, dhSize); + } + else { + DataHeaderSimple dh; + dh.type = isDelete ? 2 : 1; + memcpy(buf.data(), &dh, dhSize); + } + memcpy(buf.data() + dhSize, key.data(), key.size()); + memcpy(buf.data() + dhSize + key.size(), data.data(), data.size()); + + FawnDS_Return ret_append = data_store_->Append(entry_id, buf); + if (ret_append != OK) { + DPRINTF(2, "FawnDS_SF::InsertEntry(): failed to write entry\n"); + pthread_rwlock_unlock(&fawnds_lock_); + return ret_append; + } + + DPRINTF(2, "FawnDS_SF::InsertEntry(): inserted entry ID=%llu\n", entry_id.as_number()); + + // update the hashtable (since the data was successfully written). + assert(entry_id.as_number(-1) != static_cast(-1)); + + if (hash_it.IsEnd()) { + FawnDS_Return ret_put = hash_table_->Put(key, entry_id); + if (ret_put != OK) { + DPRINTF(2, "FawnDS_SF::InsertEntry(): failed to add to hashtable\n"); + pthread_rwlock_unlock(&fawnds_lock_); + // TODO: at this point, the log contains data but the hash table doesn't. + // will this make inconsistency (with/without index reloading) + // i.e. merging process will consistently lose one entry. + return ret_put; + } + } + else { + FawnDS_Return ret_replace = hash_it->Replace(entry_id); + if (ret_replace != OK) { + DPRINTF(2, "FawnDS_SF::InsertEntry(): failed to update hashtable\n"); + pthread_rwlock_unlock(&fawnds_lock_); + // TODO: this must be rare case (maybe a bug), but this also has potential inconsistency issues + return ret_replace; + } + } + + // Update number of elements if Insert added a new object + if (!isDelete) { + if (newObj) + header_->num_active_elements++; + } + else { + // count a delete entry as an active element + /* + if (!newObj) { + header_->num_active_elements--; + } + */ + } + header_->num_elements++; + + DPRINTF(2, "FawnDS_SF::InsertEntry(): updated hashtable\n"); + + pthread_rwlock_unlock(&fawnds_lock_); + + if (!isDelete) + return OK; + else { + if (!newObj) + return OK; + else + return KEY_NOT_FOUND; + } +} + +FawnDS_Return +FawnDS_SF::Contains(const ConstValue& key) const +{ + if (key.size() == 0) + return INVALID_KEY; + + if (key_len_ != 0 && key_len_ != key.size()) + return INVALID_KEY; + + // TODO: fix this naive implementation + size_t len = 0; + FawnDS_Return ret = Length(key, len); + if (ret == OK) { + return OK; + } + else + return KEY_NOT_FOUND; +} + +FawnDS_Return +FawnDS_SF::Length(const ConstValue& key, size_t& len) const +{ +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FawnDS_SF::Length(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + } +#endif + + if (key.size() == 0) + return INVALID_KEY; + + if (key_len_ != 0 && key_len_ != key.size()) + return INVALID_KEY; + + // TODO: implement this in a better way + SizedValue<4096> data; + FawnDS_Return ret_get = Get(key, data); + + len = data.size(); +#ifdef DEBUG + if (debug_level & 2) + DPRINTF(2, "FawnDS_SF::Length(): len=%zu\n", len); +#endif + return ret_get; +} + +FawnDS_Return +FawnDS_SF::Get(const ConstValue& key, Value& data, size_t offset, size_t len) const +{ + if (key.size() == 0) + return INVALID_KEY; + + if (key_len_ != 0 && key_len_ != key.size()) + return INVALID_KEY; + +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FawnDS_SF::Get(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + } +#endif + + pthread_rwlock_rdlock(&fawnds_lock_); + + bool newObj = true; + + FawnDS_Iterator hash_it = hash_table_->Find(key); + size_t dhSize; + if (key_len_ == 0) + dhSize = sizeof(DataHeaderFull); + else + dhSize = sizeof(DataHeaderSimple); + + // Look for an existing entry in the hashtable + // TODO: this code seems to almost redundant to InsertEntry(); any way to simplify? + Value data_store_key; + SizedValue<64> read_hdr; + while (!hash_it.IsEnd()) { + data_store_key = hash_it->data; + + if (data_store_->Get(data_store_key, read_hdr, 0, dhSize) != OK || + (key_len_ == 0 && read_hdr.as().key_len != key.size())) { + DPRINTF(2, "FawnDS_SF::Get(): key length mismatch; continue\n"); + ++hash_it; + continue; + } + + DPRINTF(2, "FawnDS_SF::Get(): key length match\n"); + + SizedValue<64> read_key; + if (data_store_->Get(data_store_key, read_key, dhSize, key.size()) != OK || + key != read_key) { + DPRINTF(2, "FawnDS_SF::Get(): key mismatch; continue\n"); + ++hash_it; + continue; + } + + DPRINTF(2, "FawnDS_SF::Get(): key match\n"); + newObj = false; + break; + } + + if (newObj) { + DPRINTF(2, "FawnDS_SF::Get(): cannot find key\n"); + pthread_rwlock_unlock(&fawnds_lock_); + return KEY_NOT_FOUND; + } + + size_t total_len = 0; + FawnDS_Return ret_len = data_store_->Length(data_store_key, total_len); + if (ret_len != OK) { + pthread_rwlock_unlock(&fawnds_lock_); + return ret_len; + } + + uint8_t type; + if (key_len_ == 0) + type = read_hdr.as().type; + else + type = read_hdr.as().type; + + if (type == 0) { + // this should not happend -- an unused entry is referenced by the hash table + assert(false); + return ERROR; + } + else if (type == 2) { + DPRINTF(2, "FawnDS_SF::Get(): deleted key\n"); + pthread_rwlock_unlock(&fawnds_lock_); + return KEY_DELETED; + } + + size_t data_len = total_len - dhSize - key.size(); + + if (offset > data_len) { + pthread_rwlock_unlock(&fawnds_lock_); + data.resize(0); + return OK; + } + + if (offset + len > data_len) + len = data_len - offset; + + FawnDS_Return ret_data = data_store_->Get(data_store_key, data, dhSize + key.size() + offset, len); + +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FawnDS_SF::Get(): data=\n"); + print_payload((const u_char*)data.data(), data.size(), 4); + } +#endif + + pthread_rwlock_unlock(&fawnds_lock_); + return ret_data; +} + +FawnDS_ConstIterator +FawnDS_SF::Enumerate() const +{ + IteratorElem* elem = new IteratorElem(this); + elem->data_store_it = data_store_->Enumerate(); + elem->Increment(true); + return FawnDS_ConstIterator(elem); +} + +FawnDS_Iterator +FawnDS_SF::Enumerate() +{ + IteratorElem* elem = new IteratorElem(this); + elem->data_store_it = data_store_->Enumerate(); + elem->Increment(true); + return FawnDS_Iterator(elem); +} + +FawnDS_SF::IteratorElem::IteratorElem(const FawnDS_SF* fawnds) +{ + this->fawnds = fawnds; + pthread_rwlock_rdlock(&static_cast(fawnds)->fawnds_lock_); +} + +FawnDS_SF::IteratorElem::~IteratorElem() +{ + pthread_rwlock_unlock(&static_cast(fawnds)->fawnds_lock_); +} + +FawnDS_IteratorElem* +FawnDS_SF::IteratorElem::Clone() const +{ + IteratorElem* elem = new IteratorElem(static_cast(fawnds)); + *elem = *this; + return elem; +} + +void +FawnDS_SF::IteratorElem::Next() +{ + Increment(false); +} + +void +FawnDS_SF::IteratorElem::Increment(bool initial) +{ + while (true) { + if (initial) + initial = false; + else + ++data_store_it; + + const FawnDS_SF* fawnds_sf = static_cast(fawnds); + + if (data_store_it.IsOK()) { + Value& data_store_data = data_store_it->data; + + size_t dhSize; + size_t key_len; + uint8_t type; + if (fawnds_sf->key_len_ == 0) { + dhSize = sizeof(DataHeaderFull); + key_len = data_store_data.as().key_len; + type = data_store_data.as().type; + } + else { + dhSize = sizeof(DataHeaderSimple); + key_len = fawnds_sf->key_len_; + type = data_store_data.as().type; + } + + if (type == 0) { + // skip unused space + continue; + } + else if (type == 1) + state = OK; + else if (type == 2) + state = KEY_DELETED; + else { + // corrupted data? + state = ERROR; + assert(false); + } + + key = NewValue(data_store_data.data() + dhSize, key_len); + data = NewValue(data_store_data.data() + dhSize + key_len, data_store_data.size() - dhSize - key_len); + break; + } + else if (data_store_it.IsEnd()) { + state = END; + break; + } + else { + state = ERROR; + break; + } + } +} + +/***************************************************/ +/************** DB Utility Functions ***************/ +/***************************************************/ + +FawnDS_Return +FawnDS_SF::WriteHeaderToFile() const { + uint64_t length = sizeof(struct DbHeader); + uint64_t offset = 0; + + string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + int fd; + if ((fd = open(filename.c_str(), O_RDWR|O_CREAT|O_NOATIME, 0666)) == -1) { + perror("Could not open header file"); + return ERROR; + } + + if (ftruncate(fd, (off_t)length) == -1) { + fprintf(stderr, "Could not extend header file to %"PRIu64" bytes: \ + %s\n", length, strerror(errno)); + } + lseek(fd, 0, SEEK_SET); + + if (fill_file_with_zeros(fd, length) < 0) { + fprintf(stderr, "Could not zero out header file.\n"); + close(fd); + return ERROR; + } + + // write the header for the hashtable + if ((uint64_t)pwrite64(fd, header_, length, offset) != length) { + fprintf(stderr, "Could not write malloc'd dbheader at position %d: \ + %s\n", 0, strerror(errno)); + close(fd); + return ERROR; + } + + fdatasync(fd); + close(fd); + return OK; +} + +// This assumes that the file was closed properly. +FawnDS_Return +FawnDS_SF::ReadHeaderFromFile() +{ + cout << "Reading header from file..." << endl; + header_ = (struct DbHeader*)malloc(sizeof(struct DbHeader)); + if (header_ == NULL) { + perror("could not malloc header file\n"); + return ERROR; + } + + string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + int fd; + if ((fd = open(filename.c_str(), O_RDWR|O_CREAT|O_NOATIME, 0666)) == -1) + { + perror("Could not open header file"); + return ERROR; + } + uint64_t length = sizeof(struct DbHeader); + uint64_t offset = 0; + if ((uint64_t)pread64(fd, header_, length, offset) != length) { + fprintf(stderr, "Could not read header for data at position \ + %"PRIu64": %s\n", offset, strerror(errno)); + return ERROR; + } + return OK; +} + +} // namespace fawn + diff --git a/fawnds/fawnds_sf.h b/fawnds/fawnds_sf.h new file mode 100644 index 0000000..b5ec5cc --- /dev/null +++ b/fawnds/fawnds_sf.h @@ -0,0 +1,118 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _FAWNDS_SF_H_ +#define _FAWNDS_SF_H_ + +#include "fawnds.h" + +namespace fawn { + + // configuration + // : "sf" (fixed) + // : the ID of the store + // : the file name prefix to store bookkeeping information for persistency + // : the length of keys -- zero for variable-length keys (default), a positive integer for fixed-length keys (space optimization can be applied in the future) + // : the length of data -- zero for variable-length data (default), a positive integer for fixed-length data (space optimization can be applied in the future) + // : omit support for deletion (not implemented yet) + // : when converting another FawnDS_SF store to this store, assume that the previous hash table preserves the same data layout in this data store; datastore->ConvertTo() is used instead of data reorganization based on side-by-side hash table comparison. + // : the configuration for the hash table + // : will be assigned by FawnDS_SF + // : the configuration for the insert-order-preserving data store + // : will be assigned by FawnDS_SF + // : will be set by FawnDS_SF (zero if either key length or data length is zero, (header size) + key length + data length otherwise) + + class FawnDS_SF : public FawnDS { + public: + FawnDS_SF(); + virtual ~FawnDS_SF(); + + virtual FawnDS_Return Create(); + virtual FawnDS_Return Open(); + + virtual FawnDS_Return ConvertTo(FawnDS* new_store) const; + + virtual FawnDS_Return Flush(); + virtual FawnDS_Return Close(); + + virtual FawnDS_Return Destroy(); + + virtual FawnDS_Return Status(const FawnDS_StatusType& type, Value& status) const; + + virtual FawnDS_Return Put(const ConstValue& key, const ConstValue& data); + //virtual FawnDS_Return Append(Value& key, const ConstValue& data); + + virtual FawnDS_Return Delete(const ConstValue& key); + + virtual FawnDS_Return Contains(const ConstValue& key) const; + virtual FawnDS_Return Length(const ConstValue& key, size_t& len) const; + virtual FawnDS_Return Get(const ConstValue& key, Value& data, size_t offset = 0, size_t len = -1) const; + + virtual FawnDS_ConstIterator Enumerate() const; + virtual FawnDS_Iterator Enumerate(); + + //virtual FawnDS_ConstIterator Find(const ConstValue& key) const; + //virtual FawnDS_Iterator Find(const ConstValue& key); + + struct IteratorElem : public FawnDS_IteratorElem { + IteratorElem(const FawnDS_SF* fawnds); + ~IteratorElem(); + + FawnDS_IteratorElem* Clone() const; + void Next(); + + void Increment(bool initial); + + FawnDS_Iterator data_store_it; + }; + + protected: + FawnDS_Return InsertEntry(const ConstValue& key, const ConstValue& data, bool isDelete); + + struct DbHeader { + uint64_t magic_number; + uint64_t num_elements; + uint64_t num_active_elements; + //double max_deleted_ratio; + //double max_load_factor; + //keyType keyFormat; + } __attribute__((__packed__)); + + struct DataHeaderFull { + uint16_t type; // 0 = unused, 1 = insert/update, 2 = delete + uint16_t key_len; + } __attribute__((__packed__)); + + struct DataHeaderSimple { + uint16_t type; // 0 = unused, 1 = insert/update, 2 = delete + uint16_t padding; + } __attribute__((__packed__)); + + /* + static const int DSReadMin = 2048; + struct DataHeaderExtended { + //uint32_t data_length; + uint32_t key_length; + bool deleteLog; + char partial_data[DSReadMin-sizeof(struct DataHeader)]; + } __attribute__((__packed__)); + */ + + private: + FawnDS_Return WriteHeaderToFile() const __attribute__ ((warn_unused_result)); + FawnDS_Return ReadHeaderFromFile() __attribute__ ((warn_unused_result)); + + DbHeader* header_; + FawnDS* hash_table_; + + FawnDS* data_store_; + + size_t key_len_; + size_t data_len_; + + mutable pthread_rwlock_t fawnds_lock_; + + friend struct IteratorElem; + }; + +} // namespace fawn + +#endif // #ifndef _FAWNDS_SF_H_ diff --git a/fawnds/fawnds_sf_ordered_binarysearch.cc b/fawnds/fawnds_sf_ordered_binarysearch.cc new file mode 100644 index 0000000..837dc54 --- /dev/null +++ b/fawnds/fawnds_sf_ordered_binarysearch.cc @@ -0,0 +1,399 @@ +#include "fawnds_sf_ordered_binarysearch.h" +#include // for debugging (hash_state()) + +namespace fawn +{ + template + FawnDS_SF_Ordered_BinarySearch* + FawnDS_SF_Ordered_BinarySearch::Create(Configuration* const &config, const uint64_t &hash_table_size) + { + FawnDS_SF_Ordered_BinarySearch* fawnds = new FawnDS_SF_Ordered_BinarySearch(config); + fawnds->Initialize(hash_table_size); + return fawnds; + } + + template + FawnDS_SF_Ordered_BinarySearch* + FawnDS_SF_Ordered_BinarySearch::Open(Configuration* const &config) + { + FawnDS_SF_Ordered_BinarySearch* fawnds = new FawnDS_SF_Ordered_BinarySearch(config); + fawnds->LoadFromFile(); + return fawnds; + } + + template + FawnDS_SF_Ordered_BinarySearch::FawnDS_SF_Ordered_BinarySearch(Configuration* const &config) + : config_(config), initialized_(false) + { + pthread_rwlock_init(&fawnds_lock_, NULL); + + // TODO: range check + key_len_ = atoi(config->getStringValue("datastore/key-len").c_str()); + data_len_ = atoi(config->getStringValue("datastore/data-len").c_str()); + + Configuration* const storeConfig = new Configuration(config_); + storeConfig->setPrefix(config_->getPrefix() + "/datastore"); + data_store_ = new Store(storeConfig); + + last_inserted_key_ = new char[key_len_]; + } + + template + FawnDS_SF_Ordered_BinarySearch::~FawnDS_SF_Ordered_BinarySearch() + { + delete [] last_inserted_key_; + delete data_store_; + + pthread_rwlock_destroy(&fawnds_lock_); + } + + template + void + FawnDS_SF_Ordered_BinarySearch::Initialize(uint64_t hash_table_size) + { + if (initialized_) { + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Initialize: reinitailization\n"); + return; + } + + base_data_store_offset_ = data_store_->getCurrentOffset(); + hash_table_size_ = hash_table_size; + actual_size_ = 0; + + finalized_ = false; + + initialized_ = true; + fprintf(stdout, "FawnDS_SF_Ordered_BinarySearch: Initialize: initialized\n"); + } + + template + void + FawnDS_SF_Ordered_BinarySearch::LoadFromFile() + { + if (initialized_) { + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: LoadFromFile: already initialized\n"); + return; + } + + string filename = config_->getStringValue("file"); + int fd; + if ((fd = open(filename.c_str(), O_RDONLY | O_NOATIME, 0666)) == -1) { + perror("FawnDS_SF_Ordered_BinarySearch: LoadFromFile: could not open index file"); + return; + } + + // read base offset + if (read(fd, &base_data_store_offset_, sizeof(base_data_store_offset_)) != sizeof(base_data_store_offset_)) { + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: LoadFromFile: unable to read base offset\n"); + close(fd); + return; + } + + // read hash table size + if (read(fd, &hash_table_size_, sizeof(hash_table_size_)) != sizeof(hash_table_size_)) { + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: LoadFromFile: unable to read hash table size\n"); + close(fd); + return; + } + + // read actual hash table size + if (read(fd, &actual_size_, sizeof(actual_size_)) != sizeof(actual_size_)) { + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: LoadFromFile: unable to read actual hash table size\n"); + close(fd); + return; + } + + close(fd); + finalized_ = true; + + initialized_ = true; + fprintf(stdout, "FawnDS_SF_Ordered_BinarySearch: LoadFromFile: loaded\n"); + +#ifdef DEBUG + fprintf(stdout, "FawnDS_SF_Ordered_BinarySearch: LoadFromFile: state: %lu\n", hash_state()); +#endif + } + + template + void + FawnDS_SF_Ordered_BinarySearch::StoreToFile() + { + if (!initialized_) { + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: StoreToFile: not initailized\n"); + return; + } + + if (!finalized_) { + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: StoreToFile: not finalized\n"); + return; + } + + string filename = config_->getStringValue("file"); + int fd; + if ((fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_NOATIME, 0666)) == -1) { + perror("FawnDS_SF_Ordered_BinarySearch: StoreToFile: could not open index file"); + return; + } + + // write base offset + if (write(fd, &base_data_store_offset_, sizeof(base_data_store_offset_)) != sizeof(base_data_store_offset_)) { + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: StoreToFile: unable to write base offset\n"); + close(fd); + return; + } + + // write hash table size + if (write(fd, &hash_table_size_, sizeof(hash_table_size_)) != sizeof(hash_table_size_)) { + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: StoreToFile: unable to write hash table size\n"); + close(fd); + return; + } + + // write actual hash table size + if (write(fd, &actual_size_, sizeof(actual_size_)) != sizeof(actual_size_)) { + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: StoreToFile: unable to write actual hash table size\n"); + close(fd); + return; + } + + close(fd); + fprintf(stdout, "FawnDS_SF_Ordered_BinarySearch: StoreToFile: stored\n"); + +#ifdef DEBUG + fprintf(stdout, "FawnDS_SF_Ordered_BinarySearch: StoreToFile: state: %lu\n", hash_state()); +#endif + } + + template + int + FawnDS_SF_Ordered_BinarySearch::Insert(const char* const &key, + const uint32_t &key_len, + const char* const &data, + const uint32_t &data_len) + { +#ifdef DEBUG + if (debug_level & 2) { + if (key_len != key_len_) + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Insert: invalid key_len\n"); + if (data_len != data_len_) + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Insert: invalid data_len\n"); + if (key == NULL) + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Insert: key is NULL!\n"); + if (data == NULL) + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Insert: data is NULL!\n"); + } +#endif + + DPRINTF(2, "FawnDS_SF_Ordered_BinarySearch: Called Insert for key with length: %i at mem: %i and data: %s with length: %i at mem: %i ...", key_len, key, data, data_len, data); + + if (key == NULL || data == NULL || key_len != key_len_ || data_len != data_len_) { + return -1; + } + + pthread_rwlock_wrlock(&fawnds_lock_); + + if (finalized_) + { + pthread_rwlock_unlock(&fawnds_lock_); + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Insert: already finalized\n"); + return -1; + } + + if (actual_size_ > 0 && memcmp(last_inserted_key_, key, key_len) >= 0) + { + pthread_rwlock_unlock(&fawnds_lock_); + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Insert: duplicate or not sorted key\n"); + return -1; + } + memcpy(last_inserted_key_, key, key_len); + + // put key-data to datastore + off_t data_store_offset = data_store_->getCurrentOffset(); + if (data_store_->Put(data_store_offset, key_len, data_len, key, data) != 0) { + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Insert: failed to Put to datastore\n"); + pthread_rwlock_unlock(&fawnds_lock_); + return -1; + } + + actual_size_++; + + pthread_rwlock_unlock(&fawnds_lock_); + DPRINTF(2, "Inserted!\n"); + + return 0; + } + + template + int + FawnDS_SF_Ordered_BinarySearch::Get(const char* const &key, const uint32_t &key_len, char* &data, uint32_t &data_len) + { +#ifdef DEBUG + if (debug_level & 2) { + if (key_len != key_len_) + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Get: invalid key_len\n"); + if (key == NULL) + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Get: key is NULL!\n"); + } +#endif + + DPRINTF(2, "FawnDS_SF_Ordered_BinarySearch: Called Get for key with length: %i at mem: %i ...", key_len, key); + + if (key == NULL || key_len != key_len_) { + return false; + } + + pthread_rwlock_rdlock(&fawnds_lock_); + + if (!finalized_) + { + pthread_rwlock_unlock(&fawnds_lock_); + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Get: not finalized\n"); + return -1; + } + + std::size_t key_index = lookup_key_index(key, 0, actual_size_); + if (key_index != static_cast(-1)) + { + // TODO: hardcoded record size + data_len = data_len_; + off_t datapos = base_data_store_offset_ + key_index * (key_len_ + data_len_); + + //printf("reading %lu\n", datapos); + data = new char[data_len]; + if(data_store_->RetrieveData(datapos, key_len, data_len, data) < 0) { + delete [] data; + pthread_rwlock_unlock(&fawnds_lock_); + return -1; + } + pthread_rwlock_unlock(&fawnds_lock_); + DPRINTF(2, "Get! Data: %s\n", data); + return 0; + } + + pthread_rwlock_unlock(&fawnds_lock_); + fprintf(stderr, "Can't find data for given key.\n"); + return -1; + } + + template + int + FawnDS_SF_Ordered_BinarySearch::Delete(const char* const &key __attribute__ ((unused)), const uint32_t &key_len __attribute__ ((unused))) + { + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Delete: not supported\n"); + return -1; + } + + /* + template + int + FawnDS_SF_Ordered_BinarySearch::DoForEach(void (*callForEach)(const uint32_t &key_len, const uint32_t &data_len, const char* const &key, const char* const &data, const bool &isDeleted), void (*callBeforeRest)()) const + { + return data_store_->DoForEach(callForEach, callBeforeRest); + } + */ + + template + int + FawnDS_SF_Ordered_BinarySearch::Flush() + { + pthread_rwlock_wrlock(&fawnds_lock_); + + if (finalized_) { + pthread_rwlock_unlock(&fawnds_lock_); + fprintf(stderr, "FawnDS_SF_Ordered_BinarySearch: Flush: already finalized\n"); + return -1; + } + + finalize(); + + if (!finalized_) + { + pthread_rwlock_unlock(&fawnds_lock_); + return -1; + } + + pthread_rwlock_unlock(&fawnds_lock_); + return 0; + } + + template + int + FawnDS_SF_Ordered_BinarySearch::Destroy() + { + return data_store_->Destroy(); + } + + template + FawnDSIter + FawnDS_SF_Ordered_BinarySearch::IteratorBegin() const { + return data_store_->IteratorBegin(); + } + + template + std::size_t + FawnDS_SF_Ordered_BinarySearch::lookup_key_index(const char* key, uint64_t off, int64_t n) const + { + //printf("binary search %lu %ld\n", off, n); + + if (n > 0) { + int64_t n2 = n / 2; + uint64_t mid = off + n2; + + uint32_t ckey_len; + uint32_t cdata_len; + bool cisDeleted; + + // TODO: hardcoded record size + off_t datapos = base_data_store_offset_ + mid * (key_len_ + data_len_); + + //printf("reading %lu\n", datapos); + if (data_store_->RetrieveMetadata(datapos, ckey_len, cdata_len, cisDeleted) == 0 && ckey_len == key_len_ && cisDeleted == false) { + char ckey[ckey_len]; + if(data_store_->RetrieveKey(datapos, key_len_, cdata_len, ckey) == 0) { + int cmp = memcmp(key, ckey, key_len_); + //printf("cmp %d\n", cmp); + if (cmp < 0) + return lookup_key_index(key, off, n2); + else if (cmp > 0) + return lookup_key_index(key, mid + 1, n - (n2 + 1)); + else + return mid; + } + } + } + + return static_cast(-1); + } + + template + void + FawnDS_SF_Ordered_BinarySearch::finalize() + { + finalized_ = true; + + StoreToFile(); + } + + template + std::size_t + FawnDS_SF_Ordered_BinarySearch::hash_state() const + { + std::size_t sum = 0; + + boost::hash_combine(sum, initialized_); + boost::hash_combine(sum, finalized_); + + boost::hash_combine(sum, key_len_); + boost::hash_combine(sum, data_len_); + + boost::hash_combine(sum, base_data_store_offset_); + boost::hash_combine(sum, hash_table_size_); + boost::hash_combine(sum, actual_size_); + + return sum; + } + + + template class FawnDS_SF_Ordered_BinarySearch; +} // namespace fawn + diff --git a/fawnds/fawnds_sf_ordered_binarysearch.h b/fawnds/fawnds_sf_ordered_binarysearch.h new file mode 100644 index 0000000..fc4cb5f --- /dev/null +++ b/fawnds/fawnds_sf_ordered_binarysearch.h @@ -0,0 +1,62 @@ +#ifndef _FAWNDS_SF_ORDERED_BINARYSEARCH_H_ +#define _FAWNDS_SF_ORDERED_BINARYSEARCH_H_ + +#include "fawnds_factory.h" + +namespace fawn +{ + template + class FawnDS_SF_Ordered_BinarySearch : public FawnDS + { + public: + static FawnDS_SF_Ordered_BinarySearch* Create(Configuration* const &config, const uint64_t &hash_table_size) __attribute__ ((warn_unused_result)); + static FawnDS_SF_Ordered_BinarySearch* Open(Configuration* const &config) __attribute__ ((warn_unused_result)); + + protected: + FawnDS_SF_Ordered_BinarySearch(Configuration* const &config); + void Initialize(uint64_t hash_table_size); + void LoadFromFile(); + void StoreToFile(); + + public: + ~FawnDS_SF_Ordered_BinarySearch(); + + int Insert(const char* const &key, const uint32_t &key_len, const char* const &data, const uint32_t &data_len) __attribute__ ((warn_unused_result)); + int Get(const char* const &key, const uint32_t &key_len, char* &data, uint32_t &data_len) __attribute__ ((warn_unused_result)); + int Delete(const char* const &key, const uint32_t &key_len) __attribute__ ((warn_unused_result)); + //int DoForEach(void (*callForEach)(const uint32_t &key_len, const uint32_t &data_len, const char* const &key, const char* const &data, const bool &isDeleted), void (*callBeforeRest)()) const __attribute__ ((warn_unused_result)); + int Flush() __attribute__ ((warn_unused_result)); + int Destroy() __attribute__ ((warn_unused_result)); + + FawnDSIter IteratorBegin() const __attribute__ ((warn_unused_result)); + + protected: + string id_; + Configuration* config_; + + Store* data_store_; + + protected: + std::size_t lookup_key_index(const char* key, uint64_t off, int64_t n) const; + + void finalize(); + + std::size_t hash_state() const; + + private: + bool initialized_; + bool finalized_; + + uint32_t key_len_; + uint32_t data_len_; + + off_t base_data_store_offset_; + uint64_t hash_table_size_; + uint64_t actual_size_; + + char* last_inserted_key_; // for validating new keys + }; +} // namespace fawn + +#endif // #ifndef _FAWNDS_SF_ORDERED_BINARYSEARCH_H_ + diff --git a/fawnds/fawnds_sf_ordered_trie.cc b/fawnds/fawnds_sf_ordered_trie.cc new file mode 100644 index 0000000..ccd9fb2 --- /dev/null +++ b/fawnds/fawnds_sf_ordered_trie.cc @@ -0,0 +1,624 @@ +#include "fawnds_sf_ordered_trie.h" +#include "debug.h" +#include "print.h" +#include + +namespace fawn +{ + FawnDS_SF_Ordered_Trie::FawnDS_SF_Ordered_Trie() + : index_(NULL), data_store_(NULL) + { + } + + FawnDS_SF_Ordered_Trie::~FawnDS_SF_Ordered_Trie() + { + if (index_) + Close(); + } + + FawnDS_Return + FawnDS_SF_Ordered_Trie::Create() + { + if (index_) + return ERROR; + + key_len_ = atoi(config_->GetStringValue("child::key-len").c_str()); + data_len_ = atoi(config_->GetStringValue("child::data-len").c_str()); + keys_per_block_ = atoi(config_->GetStringValue("child::keys-per-block").c_str()); + + size_ = atoi(config_->GetStringValue("child::size").c_str()); + bucket_size_ = atoi(config_->GetStringValue("child::bucket-size").c_str()); + + skip_bits_ = atoi(config_->GetStringValue("child::skip-bits").c_str()); + + actual_size_ = 0; + + if (key_len_ == 0) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie::Create(): invalid key length\n"); + return ERROR; + } + + if (data_len_ == 0) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie::Create(): invalid data length\n"); + return ERROR; + } + + if (keys_per_block_ == 0) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie::Create(): invalid keys per block\n"); + return ERROR; + } + + // the size can be zero + /* + if (size_ == 0) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Create(): invalid size\n"); + return ERROR; + } + */ + + if (bucket_size_ == 0) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie::Create(): invalid bucket size\n"); + return ERROR; + } + + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Create(): creating index\n"); + + index_ = new index_type(key_len_, size_, bucket_size_, 0, keys_per_block_, skip_bits_); + + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Create(): creating data store\n"); + + Configuration* storeConfig = new Configuration(config_, true); + storeConfig->SetContextNode("child::datastore"); + storeConfig->SetStringValue("child::id", config_->GetStringValue("child::id")); + char buf[1024]; + //snprintf(buf, sizeof(buf), "%zu", key_len_ + data_len_); + snprintf(buf, sizeof(buf), "%zu", key_len_ + data_len_ + 4); // HACK: alignment for 1020-byte entry + storeConfig->SetStringValue("child::data-len", buf); + data_store_ = FawnDS_Factory::New(storeConfig); + if (data_store_ == NULL) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Create(): could not create data store\n"); + return ERROR; + } + + FawnDS_Return ret = data_store_->Create(); + if (ret != OK) + return ret; + + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Create(): done\n"); + + return OK; + } + + FawnDS_Return + FawnDS_SF_Ordered_Trie::Open() + { + // TODO: implement + return ERROR; + } + + FawnDS_Return + FawnDS_SF_Ordered_Trie::Flush() + { + if (!index_) + return ERROR; + + if (!index_->finalized()) + index_->flush(); + else { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Flush(): already finalized\n"); + } + + return OK; + } + + FawnDS_Return + FawnDS_SF_Ordered_Trie::Close() + { + if (!index_) + return OK; + + Flush(); + + delete index_; + index_ = NULL; + delete data_store_; + data_store_ = NULL; + + return OK; + } + + FawnDS_Return + FawnDS_SF_Ordered_Trie::Destroy() + { + Configuration* storeConfig = new Configuration(config_, true); + storeConfig->SetContextNode("child::datastore"); + storeConfig->SetStringValue("child::id", config_->GetStringValue("child::id")); + FawnDS* data_store = FawnDS_Factory::New(storeConfig); + FawnDS_Return ret_destroy_data_store = data_store->Destroy(); + delete data_store; + + return ret_destroy_data_store; + } + + FawnDS_Return + FawnDS_SF_Ordered_Trie::Status(const FawnDS_StatusType& type, Value& status) const + { + if (!index_) + return ERROR; + + std::ostringstream oss; + switch (type) { + case NUM_DATA: + oss << actual_size_; + break; + case NUM_ACTIVE_DATA: + oss << actual_size_; + break; + case CAPACITY: + oss << size_; + break; + case MEMORY_USE: + oss << '[' << (index_->bit_size() + 7) / 8 << ",0]"; + break; + case DISK_USE: + { + Value status_part; + if (data_store_->Status(type, status_part) != OK) + return UNSUPPORTED; + oss << "[0," << status_part.str() << ']'; + } + break; + default: + return UNSUPPORTED; + } + status = NewValue(oss.str()); + return OK; + } + + FawnDS_Return + FawnDS_SF_Ordered_Trie::Put(const ConstValue& key, const ConstValue& data) + { +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Put(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Put(): data=\n"); + print_payload((const u_char*)data.data(), data.size(), 4); + } +#endif + + if (!index_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Put(): not initialized\n"); + return ERROR; + } + + if (index_->finalized()) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Put(): already finalized\n"); + return ERROR; + } + + if (key.size() != key_len_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Put(): key length mismatch\n"); + return INVALID_KEY; + } + if (data.size() != data_len_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Put(): data length mismatch\n"); + return INVALID_DATA; + } + + if (!index_->insert(reinterpret_cast(key.data()))) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Put(): cannot insert key to index, probably non-sorted key\n"); + return INVALID_KEY; + } + + Value combined_kv; + //combined_kv.resize(key_len_ + data_len_); + combined_kv.resize(key_len_ + data_len_ + 4); // HACK: alignment for 1020-byte entry + memcpy(combined_kv.data(), key.data(), key_len_); + memcpy(combined_kv.data() + key_len_, data.data(), data_len_); + memset(combined_kv.data() + key_len_ + data_len_, 0, 4); // HACK: alignment for 1020-byte entry + + SizedValue<64> data_store_key; + data_store_->Append(data_store_key, combined_kv); + + actual_size_++; + + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Put(): put new key\n"); + return OK; + } + + FawnDS_Return + FawnDS_SF_Ordered_Trie::Contains(const ConstValue& key) const + { + if (!index_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Contains(): not initialized\n"); + return ERROR; + } + + if (!index_->finalized()) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Contains(): not finalized\n"); + return ERROR; + } + + if (key.size() != key_len_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Contains(): key length mismatch\n"); + return INVALID_KEY; + } + + // TODO: better implementation + SizedValue<64> data; + FawnDS_Return ret_get = Get(key, data); + return ret_get; + } + + FawnDS_Return + FawnDS_SF_Ordered_Trie::Length(const ConstValue& key, size_t& len) const + { + if (!index_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Length(): not initialized\n"); + return ERROR; + } + + if (!index_->finalized()) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Length(): not finalized\n"); + return ERROR; + } + + if (key.size() != key_len_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Length(): key length mismatch\n"); + return INVALID_KEY; + } + + // TODO: better implementation + len = data_len_; + return Contains(key); + } + + FawnDS_Return + FawnDS_SF_Ordered_Trie::Get(const ConstValue& key, Value& data, size_t offset, size_t len) const + { + if (!index_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Get(): not initialized\n"); + return ERROR; + } + + if (!index_->finalized()) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Get(): not finalized\n"); + return ERROR; + } + + if (key.size() == 0) + return INVALID_KEY; + + if (key.size() != key_len_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Get(): key length mismatch\n"); + return INVALID_KEY; + } + +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Get(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + } +#endif + + size_t base_idx = index_->locate(reinterpret_cast(key.data())); + base_idx = base_idx / keys_per_block_ * keys_per_block_; + + for (size_t i = base_idx; i < base_idx + keys_per_block_; i++) { + + size_t data_store_key = i; + SizedValue<64> ret_key; + + FawnDS_Return ret_get = data_store_->Get(RefValue(&data_store_key), ret_key, 0, key_len_); + + if (ret_get == END) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Get(): got END, corrupted file store?\n"); + break; + } + + if (ret_get != OK) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Get(): got non-OK, corrupted file store?\n"); + //fprintf(stderr, "%zu error %zu\n", base_idx, ret_get); + return ret_get; + } + if (ret_key.size() != key_len_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Get(): got wrong key size, corrupted file store?\n"); + return ERROR; + } + + if (key != ret_key) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Get(): key mismatch; continue\n"); + continue; + } + + if (offset > data_len_) + return END; + + if (offset + len > data_len_) + len = data_len_ - offset; + + ret_get = data_store_->Get(RefValue(&data_store_key), data, key_len_, data_len_); + + if (ret_get != OK) { + //fprintf(stderr, "%zu error %zu\n", base_idx, ret_get); + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Get(): got non-OK, corrupted file store?\n"); + return ret_get; + } + if (data.size() != data_len_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Get(): got wrong data size, corrupted file store?\n"); + return ERROR; + } +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Get(): data=\n"); + print_payload((const u_char*)data.data(), data.size(), 4); + } +#endif + //fprintf(stderr, "%zu found\n", base_idx); + return OK; + } + + //fprintf(stderr, "%zu not found\n", base_idx); + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Get(): cannot find key\n"); + return KEY_NOT_FOUND; + } + + FawnDS_ConstIterator + FawnDS_SF_Ordered_Trie::Enumerate() const + { + if (!index_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Enumerate(): not initialized\n"); + return FawnDS_ConstIterator(); + } + + if (!index_->finalized()) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Enumerate(): not finalized\n"); + return FawnDS_ConstIterator(); + } + + IteratorElem* elem = new IteratorElem(this); + elem->data_store_it = data_store_->Enumerate(); + elem->Parse(); + return FawnDS_ConstIterator(elem); + } + + FawnDS_Iterator + FawnDS_SF_Ordered_Trie::Enumerate() + { + if (!index_) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Enumerate(): not initialized\n"); + return FawnDS_Iterator(); + } + + if (!index_->finalized()) { + DPRINTF(2, "FawnDS_SF_Ordered_Trie::Enumerate(): not finalized\n"); + return FawnDS_Iterator(); + } + + IteratorElem* elem = new IteratorElem(this); + elem->data_store_it = data_store_->Enumerate(); + elem->Parse(); + return FawnDS_Iterator(elem); + } + + FawnDS_SF_Ordered_Trie::IteratorElem::IteratorElem(const FawnDS_SF_Ordered_Trie* fawnds) + { + this->fawnds = fawnds; + } + + FawnDS_IteratorElem* + FawnDS_SF_Ordered_Trie::IteratorElem::Clone() const + { + IteratorElem* elem = new IteratorElem(static_cast(fawnds)); + *elem = *this; + return elem; + } + + void + FawnDS_SF_Ordered_Trie::IteratorElem::Next() + { + ++data_store_it; + + Parse(); + } + + void + FawnDS_SF_Ordered_Trie::IteratorElem::Parse() + { + if (data_store_it.IsOK()) { + state = OK; + const FawnDS_SF_Ordered_Trie* fawnds = static_cast(this->fawnds); + + Value& data_store_data = data_store_it->data; + + key = NewValue(data_store_data.data(), fawnds->key_len_); + data = NewValue(data_store_data.data() + fawnds->key_len_, fawnds->data_len_); + } + else if (data_store_it.IsEnd()) + state = END; + else + state = ERROR; + } + + /* + template + void + FawnDS_SF_Ordered_Trie::LoadFromFile() + { + if (initialized_) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: LoadFromFile: already initialized\n"); + return; + } + + string filename = config_->getStringValue("file"); + int fd; + if ((fd = open(filename.c_str(), O_RDONLY | O_NOATIME, 0666)) == -1) { + perror("FawnDS_SF_Ordered_Trie: LoadFromFile: could not open index file"); + return; + } + + // read base offset + if (read(fd, &base_data_store_offset_, sizeof(base_data_store_offset_)) != sizeof(base_data_store_offset_)) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: LoadFromFile: unable to read base offset\n"); + close(fd); + return; + } + + // read hash table size + if (read(fd, &hash_table_size_, sizeof(hash_table_size_)) != sizeof(hash_table_size_)) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: LoadFromFile: unable to read hash table size\n"); + close(fd); + return; + } + + // read actual hash table size (inferrable, but for compatiblity with binary search) + if (read(fd, &actual_size_, sizeof(actual_size_)) != sizeof(actual_size_)) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: LoadFromFile: unable to read actual hash table size\n"); + close(fd); + return; + } + + group_bits_ = 0; + while ((1u << (++group_bits_)) * group_size_ < hash_table_size_); + + // read hash function offsets + for (size_t i = 0; i < (1u << group_bits_) + 1; i++) { + hash_func_index_type offset; + if (read(fd, &offset, sizeof(offset)) != sizeof(offset)) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: LoadFromFile: unable to read offset\n"); + close(fd); + return; + } + hash_func_offsets_.push_back(offset); + } + + // read datastore offsets + for (size_t i = 0; i < (1u << group_bits_) + 1; i++) { + data_store_index_type offset; + if (read(fd, &offset, sizeof(offset)) != sizeof(offset)) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: LoadFromFile: unable to read offset\n"); + close(fd); + return; + } + data_store_offsets_.push_back(offset); + } + + // read hash function + hash_func_index_type end = hash_func_offsets_[(1u << group_bits_)]; + uint8_t b; + for (size_t i = 0; i < end; i += sizeof(b) * 8) { + if (read(fd, &b, sizeof(b)) != sizeof(b)) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: LoadFromFile: unable to read hash function byte\n"); + close(fd); + return; + } + for (size_t j = 0; j < sizeof(b) * 8 && i + j < end; j++) + hash_funcs_.push_back(trie::bit_access::get(&b, j)); + } + + close(fd); + finalized_ = true; + + initialized_ = true; + fprintf(stdout, "FawnDS_SF_Ordered_Trie: LoadFromFile: loaded\n"); + +#ifdef DEBUG + fprintf(stdout, "FawnDS_SF_Ordered_Trie: LoadFromFile: state: %lu\n", hash_state()); +#endif + } + + template + void + FawnDS_SF_Ordered_Trie::StoreToFile() + { + if (!initialized_) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: StoreToFile: not initailized\n"); + return; + } + + if (!finalized_) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: StoreToFile: not finalized\n"); + return; + } + + string filename = config_->getStringValue("file"); + int fd; + if ((fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_NOATIME, 0666)) == -1) { + perror("FawnDS_SF_Ordered_Trie: StoreToFile: could not open index file"); + return; + } + + // write base offset + if (write(fd, &base_data_store_offset_, sizeof(base_data_store_offset_)) != sizeof(base_data_store_offset_)) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: StoreToFile: unable to write base offset\n"); + close(fd); + return; + } + + // write hash table size + if (write(fd, &hash_table_size_, sizeof(hash_table_size_)) != sizeof(hash_table_size_)) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: StoreToFile: unable to write hash table size\n"); + close(fd); + return; + } + + // write actual hash table size + if (write(fd, &actual_size_, sizeof(actual_size_)) != sizeof(actual_size_)) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: StoreToFile: unable to write actual hash table size\n"); + close(fd); + return; + } + + // write hash function offsets + assert(hash_func_offsets_.size() == (1u << group_bits_) + 1); + for (size_t i = 0; i < (1u << group_bits_) + 1; i++) { + hash_func_index_type offset = hash_func_offsets_[i]; + if (write(fd, &offset, sizeof(offset)) != sizeof(offset)) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: StoreToFile: unable to write offset\n"); + close(fd); + return; + } + } + + // write datastore offsets + assert(data_store_offsets_.size() == (1u << group_bits_) + 1); + for (size_t i = 0; i < (1u << group_bits_) + 1; i++) { + data_store_index_type offset = data_store_offsets_[i]; + if (write(fd, &offset, sizeof(offset)) != sizeof(offset)) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: StoreToFile: unable to write offset\n"); + close(fd); + return; + } + } + + // write hash function + hash_func_index_type end = hash_func_offsets_[(1u << group_bits_)]; + assert(hash_funcs_.size() == end); + uint8_t b; + for (size_t i = 0; i < end; i += sizeof(b) * 8) { + b = 0; + for (size_t j = 0; j < sizeof(b) * 8 && i + j < end; j++) + { + if (hash_funcs_[i + j]) + trie::bit_access::set(&b, j); + else + trie::bit_access::reset(&b, j); + } + if (write(fd, &b, sizeof(b)) != sizeof(b)) { + fprintf(stderr, "FawnDS_SF_Ordered_Trie: StoreToFile: unable to write hash function byte\n"); + close(fd); + return; + } + } + + close(fd); + fprintf(stdout, "FawnDS_SF_Ordered_Trie: StoreToFile: stored\n"); + +#ifdef DEBUG + fprintf(stdout, "FawnDS_SF_Ordered_Trie: StoreToFile: state: %lu\n", hash_state()); +#endif + } + */ + +} // namespace fawn diff --git a/fawnds/fawnds_sf_ordered_trie.h b/fawnds/fawnds_sf_ordered_trie.h new file mode 100644 index 0000000..d3af09c --- /dev/null +++ b/fawnds/fawnds_sf_ordered_trie.h @@ -0,0 +1,83 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _FAWNDS_SF_ORDERED_TRIE_H_ +#define _FAWNDS_SF_ORDERED_TRIE_H_ + +#include "fawnds_factory.h" +#include "bucketing_index.hpp" +#include "twolevel_absoff_bucketing.hpp" + +namespace fawn +{ + // configuration + // : "sf_ordered_trie" (fixed) + // : the ID of the store + // : the fixed length of all keys + // : the fixed length of all data + // : the number of key-value pairs in a I/O block + // : the total number of entries in the store + // : the number of entires in each bucket + // : the number of MSBs to ignore when calculating the bucket number + // : the configuration for the data store + // : will be assigned by FawnDS_SF_Ordered_Trie + // : will be set by FawnDS_SF_Ordered_Trie + + class FawnDS_SF_Ordered_Trie : public FawnDS + { + public: + FawnDS_SF_Ordered_Trie(); + virtual ~FawnDS_SF_Ordered_Trie(); + + virtual FawnDS_Return Create(); + virtual FawnDS_Return Open(); + + virtual FawnDS_Return Flush(); + virtual FawnDS_Return Close(); + + virtual FawnDS_Return Destroy(); + + virtual FawnDS_Return Status(const FawnDS_StatusType& type, Value& status) const; + + virtual FawnDS_Return Put(const ConstValue& key, const ConstValue& data); + //virtual FawnDS_Return Append(Value& key, const ConstValue& data); + + //virtual FawnDS_Return Delete(const ConstValue& key); + + virtual FawnDS_Return Contains(const ConstValue& key) const; + virtual FawnDS_Return Length(const ConstValue& key, size_t& len) const; + virtual FawnDS_Return Get(const ConstValue& key, Value& data, size_t offset = 0, size_t len = -1) const; + + virtual FawnDS_ConstIterator Enumerate() const; + virtual FawnDS_Iterator Enumerate(); + + //virtual FawnDS_ConstIterator Find(const ConstValue& key) const; + //virtual FawnDS_Iterator Find(const ConstValue& key); + + struct IteratorElem : public FawnDS_IteratorElem { + IteratorElem(const FawnDS_SF_Ordered_Trie* fawnds); + + FawnDS_IteratorElem* Clone() const; + void Next(); + void Parse(); + + FawnDS_Iterator data_store_it; + }; + + private: + typedef cindex::bucketing_index > index_type; + index_type* index_; + FawnDS* data_store_; + + size_t key_len_; + size_t data_len_; + size_t keys_per_block_; + + size_t size_; + size_t bucket_size_; + + size_t skip_bits_; + + size_t actual_size_; + }; +} // namespace fawn + +#endif // #ifndef _FAWNDS_SF_ORDERED_TRIE_H_ diff --git a/fawnds/fawnds_types.h b/fawnds/fawnds_types.h new file mode 100644 index 0000000..9b84ff2 --- /dev/null +++ b/fawnds/fawnds_types.h @@ -0,0 +1,30 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _FAWNDS_TYPES_H_ +#define _FAWNDS_TYPES_H_ + +namespace fawn { + + enum FawnDS_Return { + OK = 0, + ERROR, + UNSUPPORTED, + KEY_DELETED, + KEY_NOT_FOUND, + INVALID_KEY, + INVALID_DATA, + INVALID_LENGTH, + INSUFFICIENT_SPACE, + END, + }; + + enum FawnDS_StatusType { + NUM_DATA = 0, // returns # of total data (including any dummy entries & holes) + NUM_ACTIVE_DATA, // returns # of total active (valid) data + CAPACITY, // returns # of total data the data store can possibly hold + MEMORY_USE, // returns memory byte size + DISK_USE, // returns disk byte size + }; + +} // namespace fawn + +#endif // #ifndef _FAWNDS_TYPES_H_ diff --git a/fawnds/file_io.cc b/fawnds/file_io.cc new file mode 100644 index 0000000..d72db80 --- /dev/null +++ b/fawnds/file_io.cc @@ -0,0 +1,42 @@ +#include "file_io.h" + +#ifdef __APPLE__ +namespace fawn { + + // non-atomic emulation of preadv and pwritev + + ssize_t + preadv(int fd, const struct iovec *iovec, int count, off_t offset) + { + ssize_t total_read_bytes = 0; + for (int i = 0; i < count; i++) { + ssize_t read_bytes = pread(fd, iovec[i].iov_base, iovec[i].iov_len, offset); + if (read_bytes < 0) + return read_bytes; + if (static_cast(read_bytes) < iovec[i].iov_len) + break; + total_read_bytes += read_bytes; + offset += iovec[i].iov_len; + } + return total_read_bytes; + } + + ssize_t + pwritev(int fd, const struct iovec *iovec, int count, off_t offset) + { + ssize_t total_written_bytes = 0; + for (int i = 0; i < count; i++) { + ssize_t written_bytes = pwrite(fd, iovec[i].iov_base, iovec[i].iov_len, offset); + if (written_bytes < 0) + return written_bytes; + if (static_cast(written_bytes) < iovec[i].iov_len) + break; + total_written_bytes += written_bytes; + offset += iovec[i].iov_len; + } + return total_written_bytes; + } + +} +#endif + diff --git a/fawnds/file_io.h b/fawnds/file_io.h new file mode 100644 index 0000000..8b456a2 --- /dev/null +++ b/fawnds/file_io.h @@ -0,0 +1,37 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _FILE_IO_H_ +#define _FILE_IO_H_ + +#ifndef _LARGEFILE64_SOURCE +#define _LARGEFILE64_SOURCE +#endif // #ifndef _LARGEFILE64_SOURCE + +#define _FILE_OFFSET_BITS 64 + +#ifndef O_NOATIME +#define O_NOATIME 0 /* O_NOATIME is linux-only */ +#endif + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 600 // for posix_fadvise() +#endif +#ifndef __USE_BSD +#define __USE_BSD // for preadv() and pwritev() +#endif + +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#define pread64 pread +#define pwrite64 pwrite +namespace fawn { + ssize_t preadv(int fd, const struct iovec *iovec, int count, off_t offset); + ssize_t pwritev(int fd, const struct iovec *iovec, int count, off_t offset); +} // namespace fawn +#endif // #ifdef __APPLE__ + +#endif // #ifndef _FILE_IO_H_ diff --git a/fawnds/file_store.cc b/fawnds/file_store.cc new file mode 100644 index 0000000..572c5e0 --- /dev/null +++ b/fawnds/file_store.cc @@ -0,0 +1,905 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ + +#include "file_store.h" +#include "file_io.h" +#include "configuration.h" +#include "debug.h" +#include "print.h" + +#include +#include +#include +#include + +#include + +namespace fawn { + + FileStore::FileStore() + { + next_append_id_ = end_id_ = 0; + next_sync_ = 0; + + int_initialize(); + } + + FileStore::~FileStore() + { + if (int_is_open()) + int_close(); + + int_terminate(); + } + + FawnDS_Return + FileStore::Create() + { + std::string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + + if (config_->ExistsNode("child::data-len") == 0) + data_len_ = atoi(config_->GetStringValue("child::data-len").c_str()); + else + data_len_ = 0; + + if (config_->ExistsNode("child::use-buffered-io-only") == 0) + use_buffered_io_only_ = atoi(config_->GetStringValue("child::use-buffered-io-only").c_str()) != 0; + else + use_buffered_io_only_ = false; + + DPRINTF(2, "FileStore::Create(): creating file: %s\n", filename.c_str()); + + if (!int_open(filename.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_NOATIME, 0666)) + return ERROR; + + next_append_id_ = end_id_ = 0; + next_sync_ = 1048576 * (1 << (rand() % 4)); + + DPRINTF(2, "FileStore::Create(): created file: %s\n", filename.c_str()); + + return OK; + } + + FawnDS_Return + FileStore::Open() + { + std::string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + + if (config_->ExistsNode("child::data-len") == 0) + data_len_ = atoi(config_->GetStringValue("child::data-len").c_str()); + else + data_len_ = 0; + + if (config_->ExistsNode("child::use-buffered-io-only") == 0) + use_buffered_io_only_ = atoi(config_->GetStringValue("child::use-buffered-io-only").c_str()) != 0; + else + use_buffered_io_only_ = false; + + DPRINTF(2, "FileStore::Open(): opening file: %s\n", filename.c_str()); + + if (!int_open(filename.c_str(), O_RDWR | O_NOATIME, 0666)) + return ERROR; + + // TODO: move this functionality to the I/O layer + next_append_id_ = end_id_ = lseek(fd_buffered_sequential_, 0, SEEK_END); // TODO: handling truncated files + next_sync_ = 1048576 * (1 << (rand() % 4)); + + DPRINTF(2, "FileStore::Open(): opened file: %s\n", filename.c_str()); + DPRINTF(2, "FileStore::Open(): next key=%llu\n", static_cast(next_append_id_)); + + return OK; + } + + FawnDS_Return + FileStore::ConvertTo(FawnDS* new_store) const + { + FileStore* file_store = dynamic_cast(new_store); + if (!file_store) + return UNSUPPORTED; + + if (data_len_ != file_store->data_len_) { + fprintf(stderr, "FileStore::ConvertTo(): data length mismatch\n"); + return ERROR; + } + + if (!file_store->int_close()) + assert(false); + + std::string src_filename = config_->GetStringValue("child::file") + "_"; + src_filename += config_->GetStringValue("child::id"); + + std::string dest_filename = file_store->config_->GetStringValue("child::file") + "_"; + dest_filename += file_store->config_->GetStringValue("child::id"); + + if (!int_unlink(dest_filename.c_str())) { + fprintf(stderr, "FileStore::ConvertTo(): cannot unlink the destination file\n"); + } + // TODO: wrap link() with int_link() + if (link(src_filename.c_str(), dest_filename.c_str())) { + fprintf(stderr, "FileStore::ConvertTo(): cannot link the destination file\n"); + } + + if (!file_store->int_open(dest_filename.c_str(), O_RDWR | O_NOATIME, 0666)) + assert(false); + + // TODO: concurrent operations + + file_store->next_append_id_ = next_append_id_; + file_store->end_id_ = end_id_; + file_store->next_sync_ = next_sync_; + + return OK; + } + + FawnDS_Return + FileStore::Flush() + { + if (!int_is_open()) + return OK; + + // TODO: do write lock to ensure all logs have been written + // just should the upper store handle concurrent accesses? + int_sync(true); + return OK; + } + + FawnDS_Return + FileStore::Close() + { + if (!int_is_open()) + return OK; + + Flush(); + + if (int_close() == -1) { + fprintf(stderr, "FileStore::Close(): Could not close file properly\n"); + return ERROR; + } + + DPRINTF(2, "FileStore::Close(): closed file\n"); + + return OK; + } + + FawnDS_Return + FileStore::Destroy() + { + std::string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + + if (!int_unlink(filename.c_str())) { + fprintf(stderr, "FileStore::Destroy(): could not delete file\n"); + return ERROR; + } + + return OK; + } + + FawnDS_Return + FileStore::Status(const FawnDS_StatusType& type, Value& status) const + { + std::ostringstream oss; + switch (type) { + case CAPACITY: + oss << -1; // unlimited + break; + case MEMORY_USE: + oss << 0; + break; + case DISK_USE: + if (data_len_ == 0) + oss << end_id_; + else + oss << end_id_ * data_len_; + break; + default: + return UNSUPPORTED; + } + status = NewValue(oss.str()); + return OK; + } + + FawnDS_Return + FileStore::Put(const ConstValue& key, const ConstValue& data) + { +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FileStore::Put(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + DPRINTF(2, "FileStore::Put(): data=\n"); + print_payload((const u_char*)data.data(), data.size(), 4); + } +#endif + + off_t id = key.as_number(-1); + if (id == -1) + return INVALID_KEY; + + off_t target = id; + if (data_len_ != 0) + target *= data_len_; + + entry_length_t entry_len = data.size(); + size_t total_len = entry_len; + if (data_len_ == 0) + total_len += sizeof(entry_length_t); + + bool wrote; + if (data_len_ == 0) { + struct iovec iov[2]; + iov[0].iov_base = &entry_len; + iov[0].iov_len = sizeof(entry_length_t); + iov[1].iov_base = const_cast(data.data()); + iov[1].iov_len = data.size(); + wrote = int_pwritev(iov, 2, target); + } + else { + struct iovec iov[1]; + iov[0].iov_base = const_cast(data.data()); + iov[0].iov_len = data.size(); + wrote = int_pwritev(iov, 1, target); + } + + if (!wrote) { + fprintf(stderr, "failed to write at file offset %llu: %s\n", static_cast(target), strerror(errno)); + return ERROR; + } + + // TODO: update correctly under concurrent writes + if (data_len_ == 0) { + if (end_id_ < static_cast(id + total_len)) + end_id_ = static_cast(id + total_len); + } + else { + if (end_id_ < id + 1) + end_id_ = id + 1; + } + + DPRINTF(2, "FileStore::Put(): data written at file offset %llu\n", static_cast(target)); + return OK; + } + + FawnDS_Return + FileStore::Append(Value& key, const ConstValue& data) + { +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FileStore::Append(): data=\n"); + print_payload((const u_char*)data.data(), data.size(), 4); + } +#endif + + if (data_len_ != 0 && data_len_ != data.size()) + return INVALID_DATA; + + off_t key_val; + + size_t append_len; + if (data_len_ == 0) { + append_len = sizeof(entry_length_t) + data.size(); + key_val = (next_append_id_ += append_len) - append_len; // atomically add a new record length, store a new value + } + else { + if (data_len_ != data.size()) + return INVALID_DATA; + append_len = data.size(); + key_val = ++next_append_id_ - 1; + } + + key = NewValue(&key_val); + + FawnDS_Return ret = Put(key, data); + if (ret != OK) { + DPRINTF(2, "FileStore::Append(): failed\n"); + return ret; + } + + // regularly clean up OS dirty buffers + if ((next_sync_ -= append_len) < 0) { + next_sync_ = 1048576 * (1 << (rand() % 4)); + int_sync(false); + } + + DPRINTF(2, "FileStore::Append(): key=%zu\n", target); + + return OK; + } + + FawnDS_Return + FileStore::Contains(const ConstValue& key) const + { + if (key.as_number(-1) >= end_id_) + return KEY_NOT_FOUND; + else + return OK; + } + + FawnDS_Return + FileStore::Length(const ConstValue& key, size_t& len) const + { + return length(key, len, false); + } + + FawnDS_Return + FileStore::length(const ConstValue& key, size_t& len, bool readahead) const + { + off_t id = key.as_number(-1); + + if (key.as_number(-1) >= end_id_) + return KEY_NOT_FOUND; + +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FileStore::Length(): key=%llu\n", static_cast(id)); + } +#endif + + if (data_len_ == 0) { + SizedValue buf; + size_t count = sizeof(entry_length_t); + buf.resize(count, false); + if (!int_pread(buf.data(), count, id, readahead)) + return ERROR; + buf.resize(count); + if (buf.size() != sizeof(entry_length_t)) + return ERROR; + len = buf.as(); + } + else + len = data_len_; + +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FileStore::Length(): length=%zu\n", len); + } +#endif + + return OK; + } + + FawnDS_Return + FileStore::Get(const ConstValue& key, Value& data, size_t offset, size_t len) const + { + return get(key, data, offset, len, false); + } + + FawnDS_Return + FileStore::get(const ConstValue& key, Value& data, size_t offset, size_t len, bool readahead) const + { + off_t id = key.as_number(-1); + + data.resize(0); + +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FileStore::Get(): key=%llu\n", static_cast(id)); + } +#endif + + if (id >= end_id_) + return END; + + size_t data_len = 0; + FawnDS_Return ret_len = length(key, data_len, readahead); + if (ret_len != OK) + return ret_len; + + if (offset > data_len) + return END; + + if (offset + len > data_len) + len = data_len - offset; + + ssize_t read_len; + + if (data_len_ == 0) { + size_t count = len; + data.resize(count, false); + if (!int_pread(data.data(), count, id + sizeof(entry_length_t) + offset, readahead)) + return ERROR; + data.resize(count); + } + else { + // try to read the entire entry to reduce I/O for the subseqeunt request + size_t count = data_len_ - offset; + data.resize(count, false); + if (!int_pread(data.data(), count, id * data_len_ + offset, readahead)) + return ERROR; + data.resize(count); + if (data.size() > len) + data.resize(len); + } + +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "FileStore::Get(): data=\n"); + print_payload((const u_char*)data.data(), data.size(), 4); + } +#endif + + return OK; + } + + FawnDS_ConstIterator + FileStore::Enumerate() const + { + IteratorElem* elem = new IteratorElem(); + elem->fawnds = this; + elem->next_id = 0; + elem->Next(); + return FawnDS_ConstIterator(elem); + } + + FawnDS_Iterator + FileStore::Enumerate() + { + IteratorElem* elem = new IteratorElem(); + elem->fawnds = this; + elem->next_id = 0; + elem->Next(); + return FawnDS_Iterator(elem); + } + + FawnDS_ConstIterator + FileStore::Find(const ConstValue& key) const + { + IteratorElem* elem = new IteratorElem(); + elem->fawnds = this; + elem->next_id = key.as_number(-1); + elem->Next(); + return FawnDS_ConstIterator(elem); + } + + FawnDS_Iterator + FileStore::Find(const ConstValue& key) + { + IteratorElem* elem = new IteratorElem(); + elem->fawnds = this; + elem->next_id = key.as_number(-1); + elem->Next(); + return FawnDS_Iterator(elem); + } + + /* + int + FileStore::disable_readahead() + { + int rc; +#ifdef __APPLE__ + int zero = 0; + if ((rc = fcntl(fd_, F_RDAHEAD, &zero)) < 0) { + perror("couldn't fcntl F_RDAHEAD"); + } +#else + if ((rc = posix_fadvise(fd_, 0, 0, POSIX_FADV_RANDOM)) < 0) { + perror("couldn't posix_fadvise random"); + } +#endif + return rc; + } + */ + + FawnDS_IteratorElem* + FileStore::IteratorElem::Clone() const + { + IteratorElem* elem = new IteratorElem(); + *elem = *this; + return elem; + } + + void + FileStore::IteratorElem::Next() + { + const FileStore* file_store = static_cast(fawnds); + + key = NewValue(&next_id); + state = file_store->get(key, data, 0, static_cast(-1), true); + + if (state == OK || state == KEY_DELETED) { + if (file_store->data_len_ == 0) + next_id += sizeof(entry_length_t) + data.size(); + else + next_id++; + } + else + state = END; + } + + void + FileStore::int_initialize() + { + fd_buffered_sequential_ = -1; + fd_buffered_random_ = -1; + fd_direct_random_ = -1; + memset(cache_, 0, sizeof(cache_entry) * cache_size_); + cache_hit_ = 0; + cache_miss_ = 0; + } + + void + FileStore::int_terminate() + { + if (int_is_open()) { + assert(false); + int_close(); + } + //fprintf(stderr, "FileStore::int_terminate(): cache hit = %zu\n", static_cast(cache_hit_)); + //fprintf(stderr, "FileStore::int_terminate(): cache miss = %zu\n", static_cast(cache_miss_)); + } + + bool + FileStore::int_open(const char* pathname, int flags, mode_t mode) + { + if (int_is_open()) + return false; + + fd_buffered_sequential_ = open(pathname, flags, mode); + if (fd_buffered_sequential_ == -1) { + fprintf(stderr, "FileStore::int_open(): cannot open file: %s: %s\n", pathname, strerror(errno)); + return false; + } + + fd_buffered_random_ = open(pathname, flags & ~(O_CREAT | ~O_TRUNC | O_RDWR | O_WRONLY) | O_RDONLY, mode); + if (fd_buffered_random_ == -1) { + fprintf(stderr, "FileStore::int_open(): cannot open file: %s: %s\n", pathname, strerror(errno)); + close(fd_buffered_sequential_); + fd_buffered_sequential_ = -1; + return false; + } + + //fprintf(stderr, "use_buffered_io_only_ = %d\n", use_buffered_io_only_); + if (!use_buffered_io_only_) + fd_direct_random_ = open(pathname, flags & ~(O_CREAT | ~O_TRUNC | O_RDWR | O_WRONLY) | O_RDONLY | O_DIRECT, mode); + else { + fd_direct_random_ = open(pathname, flags & ~(O_CREAT | ~O_TRUNC | O_RDWR | O_WRONLY) | O_RDONLY, mode); + } + if (fd_direct_random_ == -1) { + fprintf(stderr, "FileStore::int_open(): cannot open file: %s: %s\n", pathname, strerror(errno)); + close(fd_buffered_sequential_); + close(fd_buffered_random_); + fd_buffered_sequential_ = -1; + fd_buffered_random_ = -1; + return false; + } + + int ret; + int zero = 0; + (void)zero; + +#ifdef __APPLE__ + ret = fcntl(fd_buffered_random_, F_RDAHEAD, &zero); +#else + ret = posix_fadvise(fd_buffered_random_, 0, 0, POSIX_FADV_RANDOM); +#endif + if (ret) { + fprintf(stderr, "FileStore::int_open(): cannot modify readahead setting for file: %s: %s\n", pathname, strerror(errno)); + close(fd_buffered_sequential_); + close(fd_buffered_random_); + close(fd_direct_random_); + fd_buffered_sequential_ = -1; + fd_buffered_random_ = -1; + fd_direct_random_ = -1; + return false; + } + +#ifdef __APPLE__ + ret = fcntl(fd_direct_random_, F_RDAHEAD, &zero); +#else + ret = posix_fadvise(fd_direct_random_, 0, 0, POSIX_FADV_RANDOM); +#endif + if (ret) { + fprintf(stderr, "FileStore::int_open(): cannot modify readahead setting for file: %s: %s\n", pathname, strerror(errno)); + close(fd_buffered_sequential_); + close(fd_buffered_random_); + close(fd_direct_random_); + fd_buffered_sequential_ = -1; + fd_buffered_random_ = -1; + fd_direct_random_ = -1; + return false; + } + + return true; + } + + bool + FileStore::int_is_open() const + { + return fd_buffered_sequential_ != -1; + } + + bool + FileStore::int_close() + { + // lock must be acquire before checking if the file is open in order to avoid a race condition with int_sync() + tbb::queuing_mutex::scoped_lock sync_lock(sync_mutex_); + + if (!int_is_open()) + return false; + + if (close(fd_buffered_sequential_) == -1) + fprintf(stderr, "FileStore::int_unlink(): cannot close file: %s\n", strerror(errno)); + if (close(fd_buffered_random_) == -1) + fprintf(stderr, "FileStore::int_unlink(): cannot close file: %s\n", strerror(errno)); + if (close(fd_direct_random_) == -1) + fprintf(stderr, "FileStore::int_unlink(): cannot close file: %s\n", strerror(errno)); + fd_buffered_sequential_ = -1; + fd_buffered_random_ = -1; + fd_direct_random_ = -1; + + dirty_chunk_.clear(); + syncing_chunk_.clear(); + + memset(cache_, 0, sizeof(cache_entry) * cache_size_); + + return true; + } + + bool + FileStore::int_unlink(const char* pathname) + { + if (unlink(pathname)) { + fprintf(stderr, "FileStore::int_unlink(): cannot unlink file: %s: %s\n", pathname, strerror(errno)); + return false; + } + return true; + } + + bool + FileStore::int_pread(char* buf, size_t& count, off_t offset, bool readahead) const + { + if (!int_is_open()) + return false; + + // extend the beginning of the read region to the latest eariler page boundary + size_t to_discard = offset & page_size_mask_; + + off_t req_offset = offset - to_discard; + size_t req_count = to_discard + count; + + // extend the end of the read region to the eariest later page boundary + req_count = (req_count + page_size_mask_) & ~page_size_mask_; + + // sanity check + assert((req_offset & page_size_mask_) == 0); + assert((req_count & page_size_mask_) == 0); + assert(req_offset <= offset); + assert(req_count >= count); + + // allocate aligned memory + void* req_buf; + if (posix_memalign(&req_buf, page_size_, req_count)) { + fprintf(stderr, "FileStore::int_pread(): cannot allocate aligned memory: %s\n", strerror(errno)); + return false; + } + //fprintf(stderr, "FileStore::int_pread(): req_buf == %p (%zu B)\n", req_buf, req_count); + + // use cache if available + bool use_cache = false; + if (req_count <= page_size_ * cache_size_) { + bool cache_available = true; + tbb::queuing_rw_mutex::scoped_lock cache_lock(cache_mutex_, false); + for (off_t current_offset = req_offset; current_offset < req_offset + req_count; current_offset += page_size_) { + size_t cache_index = (current_offset / page_size_) % cache_size_; + if (!cache_[cache_index].valid || + cache_[cache_index].offset != current_offset) { + cache_available = false; + break; + } + } + if (cache_available) { + use_cache = true; + char* p = static_cast(req_buf); + for (off_t current_offset = req_offset; current_offset < req_offset + req_count; current_offset += page_size_) { + size_t cache_index = (current_offset / page_size_) % cache_size_; + memcpy(p, cache_[cache_index].data, page_size_); + p += page_size_; + } + //++cache_hit_; + } + else { + //++cache_miss_; + } + } + + ssize_t read_len; + if (use_cache) { + read_len = req_count; + } + else { + // choose an appropriate file descriptor + int fd; + { + tbb::queuing_rw_mutex::scoped_lock dirty_chunk_lock(dirty_chunk_mutex_, false); + + if (readahead) + fd = fd_buffered_sequential_; + else { + if (req_offset / chunk_size_ < dirty_chunk_.size() && + dirty_chunk_[req_offset / chunk_size_]) { + // the chunk is marked as dirty -- direct I/O cannot be used + fd = fd_buffered_random_; + } + else + fd = fd_direct_random_; + } + } + + // do read + //fprintf(stderr, "for %zu,%zu, requested %zu,%zu\n", offset, count, req_offset, req_count); + read_len = pread(fd, req_buf, req_count, req_offset); + + if (read_len < 0) { + fprintf(stderr, "FileStore::int_pread(): cannot read: %s\n", strerror(errno)); + return false; + } + + // put to cache + if (req_count <= page_size_ * cache_size_) { + tbb::queuing_rw_mutex::scoped_lock cache_lock(cache_mutex_, true); + char* p = static_cast(req_buf); + for (off_t current_offset = req_offset; current_offset < req_offset + (read_len - (read_len & page_size_mask_)); current_offset += page_size_) { + size_t cache_index = (current_offset / page_size_) % cache_size_; + cache_[cache_index].valid = true; + cache_[cache_index].offset = current_offset; + memcpy(cache_[cache_index].data, p, page_size_); + p += page_size_; + } + } + } + + if (read_len < to_discard) { + // no useful data read + //fprintf(stderr, "none read\n"); + count = 0; + } + else if (read_len < to_discard + count) { + // partially read + //fprintf(stderr, "partially read, discarding first %zu and last %zu\n", to_discard, count - to_discard - (read_len - to_discard)); + memcpy(buf, static_cast(req_buf) + to_discard, read_len - to_discard); + count = read_len - to_discard; + } + else { + // fully read + //fprintf(stderr, "fully read, discarding first %zu and last %zu\n", to_discard, count - to_discard - (len)); + memcpy(buf, static_cast(req_buf) + to_discard, count); + } + + free(req_buf); + + return true; + } + + bool + FileStore::int_pwritev(const struct iovec* iov, int count, off_t offset) + { + if (!int_is_open()) + return false; + + size_t write_size = 0; + for (int i = 0; i < count; i++) + write_size += iov[i].iov_len; + + size_t start_chunk = offset / chunk_size_; + size_t end_chunk = (offset + write_size + chunk_size_ - 1) / chunk_size_ + 1; + + // mark affected chunks dirty and unmark syncing chunks + { + tbb::queuing_rw_mutex::scoped_lock dirty_chunk_lock(dirty_chunk_mutex_, false); + + for (size_t chunk = start_chunk; chunk < end_chunk; chunk++) { + if (chunk < dirty_chunk_.size()) + dirty_chunk_[chunk] = true; + if (chunk < syncing_chunk_.size()) + syncing_chunk_[chunk] = false; + } + } + + // do write + ssize_t written_len = pwritev(fd_buffered_sequential_, iov, count, offset); + + // mark affected chunks dirty + { + tbb::queuing_rw_mutex::scoped_lock dirty_chunk_lock(dirty_chunk_mutex_, true); + + if (end_chunk <= dirty_chunk_.size()) { + for (size_t chunk = start_chunk; chunk < end_chunk; chunk++) + dirty_chunk_[chunk] = true; + } + else { + for (size_t chunk = start_chunk; chunk < dirty_chunk_.size(); chunk++) + dirty_chunk_[chunk] = true; + dirty_chunk_.resize(end_chunk, true); + } + } + + // invalidate cache + { + tbb::queuing_rw_mutex::scoped_lock cache_lock(cache_mutex_, true); + off_t aligned_offset = offset - (offset & page_size_mask_); + for (off_t current_offset = aligned_offset; current_offset < offset + written_len; current_offset += page_size_) { + size_t cache_index = (current_offset / page_size_) % cache_size_; + if (cache_[cache_index].valid && cache_[cache_index].offset == current_offset) + cache_[cache_index].valid = false; + } + } + + if (written_len < write_size) { + fprintf(stderr, "FileStore::int_pwritev(): cannot write: %s\n", strerror(errno)); + return false; + } + + return true; + } + + bool + FileStore::int_sync(bool blocking) + { + // write all buffered writes to flash/disk + // note: this does not block (i.e. asynchronous version of [synchronization of write buffer]) + + // lock must be acquire before checking if the file is open in order to avoid a race condition with int_close() + tbb::queuing_mutex::scoped_lock* sync_lock = new tbb::queuing_mutex::scoped_lock(); + + if (blocking) + sync_lock->acquire(sync_mutex_); + else { + if (!sync_lock->try_acquire(sync_mutex_)) { + // another syncing is ongoing + delete sync_lock; + return true; + } + } + + if (!int_is_open()) { + delete sync_lock; + return false; + } + + if (dirty_chunk_.none()) { + delete sync_lock; + return true; + } + + SyncTask* t = new SyncTask(); + t->file_store = this; + t->sync_lock = sync_lock; + task_scheduler_sync_.enqueue_task(t); + + if (blocking) { + // wait for syncing to complete + tbb::queuing_mutex::scoped_lock blocking_sync_lock(sync_mutex_); + } + + return true; + } + + FileStore::SyncTask::~SyncTask() + { + delete sync_lock; + sync_lock = NULL; + } + + void + FileStore::SyncTask::Run() + { + { + tbb::queuing_rw_mutex::scoped_lock dirty_chunk_lock(file_store->dirty_chunk_mutex_, false); + file_store->syncing_chunk_ = file_store->dirty_chunk_; + } + + if (fdatasync(file_store->fd_buffered_sequential_)) { + fprintf(stderr, "FileStore::int_pread(): cannot sync: %s\n", strerror(errno)); + return; + } + + { + tbb::queuing_rw_mutex::scoped_lock dirty_chunk_lock(file_store->dirty_chunk_mutex_, true); + file_store->syncing_chunk_.resize(file_store->dirty_chunk_.size(), false); + file_store->dirty_chunk_ -= file_store->syncing_chunk_; + } + } + + TaskScheduler FileStore::task_scheduler_sync_(1, 100); + +} // namespace fawn diff --git a/fawnds/file_store.h b/fawnds/file_store.h new file mode 100644 index 0000000..8ce0f6c --- /dev/null +++ b/fawnds/file_store.h @@ -0,0 +1,130 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _FILE_STORE_H_ +#define _FILE_STORE_H_ + +#include "fawnds.h" +#include "file_io.h" // for iovec +#include "task.h" +#include +#include +#include +#include + +namespace fawn { + + // configuration + // : "file" (fixed) + // : the ID of the file store + // : the file name prefix to store log entries for persistence + // : the length of data -- zero for variable-length data (default), a positive integer for fixed-length data (space optimization is applied) + // : with a non-zero value, use buffered I/O only and do not use direct I/O. Default is 0 (false). Useful for quick tests or data-len >= 4096 (direct I/O is less likely to improve read performance). + + class FileStore : public FawnDS { + public: + FileStore(); + virtual ~FileStore(); + + virtual FawnDS_Return Create(); + virtual FawnDS_Return Open(); + + virtual FawnDS_Return ConvertTo(FawnDS* new_store) const; + + virtual FawnDS_Return Flush(); + virtual FawnDS_Return Close(); + + virtual FawnDS_Return Destroy(); + + virtual FawnDS_Return Status(const FawnDS_StatusType& type, Value& status) const; + + virtual FawnDS_Return Put(const ConstValue& key, const ConstValue& data); + virtual FawnDS_Return Append(Value& key, const ConstValue& data); + + //virtual FawnDS_Return Delete(const ConstValue& key); + + virtual FawnDS_Return Contains(const ConstValue& key) const; + virtual FawnDS_Return Length(const ConstValue& key, size_t& len) const; + virtual FawnDS_Return Get(const ConstValue& key, Value& data, size_t offset = 0, size_t len = -1) const; + + virtual FawnDS_ConstIterator Enumerate() const; + virtual FawnDS_Iterator Enumerate(); + + virtual FawnDS_ConstIterator Find(const ConstValue& key) const; + virtual FawnDS_Iterator Find(const ConstValue& key); + + struct IteratorElem : public FawnDS_IteratorElem { + FawnDS_IteratorElem* Clone() const; + void Next(); + + off_t next_id; + }; + + protected: + typedef uint32_t entry_length_t; + + FawnDS_Return length(const ConstValue& key, size_t& len, bool readahead) const; + FawnDS_Return get(const ConstValue& key, Value& data, size_t offset, size_t len, bool readahead) const; + + //int disable_readahead(); + + // I/O layer (TODO: make this as a separate class) + void int_initialize(); + void int_terminate(); + + bool int_open(const char* pathname, int flags, mode_t mode); + bool int_is_open() const; + bool int_close(); + static bool int_unlink(const char* pathname); + + bool int_pread(char* buf, size_t& count, off_t offset, bool readahead) const; + bool int_pwritev(const struct iovec* iov, int count, off_t offset); + bool int_sync(bool blocking); + + private: + size_t data_len_; + + tbb::atomic next_append_id_; + off_t end_id_; + tbb::atomic next_sync_; + + bool use_buffered_io_only_; + + // I/O layer (TODO: make this as a separate class) + static const size_t chunk_size_ = 1048576; + static const size_t page_size_ = 512; + static const size_t page_size_mask_ = page_size_ - 1; + static const size_t cache_size_ = 128; + + int fd_buffered_sequential_; + int fd_buffered_random_; + int fd_direct_random_; + + mutable tbb::queuing_mutex sync_mutex_; + mutable tbb::queuing_rw_mutex dirty_chunk_mutex_; + boost::dynamic_bitset<> dirty_chunk_; + boost::dynamic_bitset<> syncing_chunk_; + + struct cache_entry { + bool valid; + off_t offset; + char data[page_size_]; + }; + mutable tbb::queuing_rw_mutex cache_mutex_; + mutable cache_entry cache_[cache_size_]; + mutable tbb::atomic cache_hit_; + mutable tbb::atomic cache_miss_; + + class SyncTask : public Task { + public: + virtual ~SyncTask(); + virtual void Run(); + FileStore* file_store; + tbb::queuing_mutex::scoped_lock* sync_lock; + }; + static TaskScheduler task_scheduler_sync_; + + friend class SyncTask; + }; + +} // namespace fawn + +#endif // #ifndef _FILE_STORE_H_ diff --git a/fawnds/global_limits.cc b/fawnds/global_limits.cc new file mode 100644 index 0000000..4f4de31 --- /dev/null +++ b/fawnds/global_limits.cc @@ -0,0 +1,72 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "global_limits.h" + +#include + +namespace fawn { + + GlobalLimits GlobalLimits::global_limits_; + + GlobalLimits::GlobalLimits() + : convert_rate_limiter_(0, 1000000000L, 1, 1), + merge_rate_limiter_(0, 1000000000L, 1, 1) + { + disabled_ = 0; + set_convert_rate(1000000000L); + set_merge_rate(1000000000L); + } + + GlobalLimits::~GlobalLimits() + { + } + + void + GlobalLimits::set_convert_rate(int64_t v) + { + convert_rate_limiter_.set_tokens(v / 100); + convert_rate_limiter_.set_max_tokens(v / 100); // allows burst of 0.01 sec + convert_rate_limiter_.set_ns_per_interval(1000000000L / v); + } + + void + GlobalLimits::set_merge_rate(int64_t v) + { + merge_rate_limiter_.set_tokens(v / 100); + merge_rate_limiter_.set_max_tokens(v / 100); // allows burst of 0.01 sec + merge_rate_limiter_.set_ns_per_interval(1000000000L / v); + } + + void + GlobalLimits::remove_convert_tokens(int64_t v) + { + if (enabled()) { + convert_rate_limiter_.remove_tokens(v); + //fprintf(stderr, "convert tokens: %lld\n", convert_rate_limiter_.tokens()); + } + } + + void + GlobalLimits::remove_merge_tokens(int64_t v) + { + if (enabled()) { + merge_rate_limiter_.remove_tokens(v); + //fprintf(stderr, "merge tokens: %lld\n", merge_rate_limiter_.tokens()); + } + } + + void + GlobalLimits::enable() + { + if (--disabled_ < 0) { + ++disabled_; + fprintf(stderr, "warning: probably too many calls to GlobalLimits::enable()\n"); + } + } + + void + GlobalLimits::disable() + { + ++disabled_; + } + +} // namespace fawn diff --git a/fawnds/global_limits.h b/fawnds/global_limits.h new file mode 100644 index 0000000..c87de8e --- /dev/null +++ b/fawnds/global_limits.h @@ -0,0 +1,40 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _GLOBAL_LIMITS_H_ +#define _GLOBAL_LIMITS_H_ + +#include "rate_limiter.h" +#include + +namespace fawn { + + class GlobalLimits { + public: + static GlobalLimits& instance() { return global_limits_; } + + void set_convert_rate(int64_t v); + void set_merge_rate(int64_t v); + + void remove_convert_tokens(int64_t v); + void remove_merge_tokens(int64_t v); + + bool enabled() const { return disabled_ == 0; } + + void enable(); + void disable(); + + protected: + GlobalLimits(); + ~GlobalLimits(); + + private: + static GlobalLimits global_limits_; + + tbb::atomic disabled_; + + RateLimiter convert_rate_limiter_; + RateLimiter merge_rate_limiter_; + }; + +} // namespace fawn + +#endif // #ifndef _GLOBAL_LIMITS_H_ diff --git a/fawnds/hash_functions.cc b/fawnds/hash_functions.cc new file mode 100644 index 0000000..8f9438b --- /dev/null +++ b/fawnds/hash_functions.cc @@ -0,0 +1,43 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "hash_functions.h" +#include "hashutil.h" + +namespace fawn { + + hashfn_t Hashes::hashes[HASH_COUNT] = { &Hashes::h1, + &Hashes::h2, + &Hashes::h3 }; + + uint32_t Hashes::h1(const void* buf, size_t len) + { + return HashUtil::BobHash(buf, len, 0xdeadbeef); + } + uint32_t Hashes::h2(const void* buf, size_t len) + { + return HashUtil::BobHash(buf, len, 0xc0ffee); + } + uint32_t Hashes::h3(const void* buf, size_t len) + { + return HashUtil::BobHash(buf, len, 0x101CA75); + } + + // These hashes assume that len > 32bits + uint32_t Hashes::nullhash1(const void* buf, size_t len) + { + if (len < 4) printf("Warning: keylength is not long enough to use null hash 1. Ensure keylength >= 32bits\n"); + return HashUtil::NullHash(buf, len, 0); + } + + uint32_t Hashes::nullhash2(const void* buf, size_t len) + { + if (len < 8) printf("Warning: keylength is not long enough to use null hash 2. Ensure keylength >= 64bits\n"); + return HashUtil::NullHash(buf, len, 4); + } + + uint32_t Hashes::nullhash3(const void* buf, size_t len) + { + if (len < 12) printf("Warning: keylength is not long enough to use null hash 3. Ensure keylength >= 96bits\n"); + return HashUtil::NullHash(buf, len, 8); + } + +} // namespace fawn diff --git a/fawnds/hash_functions.h b/fawnds/hash_functions.h new file mode 100644 index 0000000..6dfdff4 --- /dev/null +++ b/fawnds/hash_functions.h @@ -0,0 +1,28 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _HASH_FUNCTIONS_H_ +#define _HASH_FUNCTIONS_H_ + +#include "basic_types.h" + +typedef uint32_t (*hashfn_t)(const void*, size_t); + +namespace fawn { + + class Hashes { + public: + static const size_t HASH_COUNT = 3; + + static uint32_t h1(const void* buf, size_t len); + static uint32_t h2(const void* buf, size_t len); + static uint32_t h3(const void* buf, size_t len); + static uint32_t nullhash1(const void* buf, size_t len); + static uint32_t nullhash2(const void* buf, size_t len); + static uint32_t nullhash3(const void* buf, size_t len); + static hashfn_t hashes[HASH_COUNT]; + private: + Hashes(); + }; + +} // namespace fawn + +#endif // #ifndef _HASH_FUNCTIONS_H_ diff --git a/fawnds/hash_table_cuckoo.cc b/fawnds/hash_table_cuckoo.cc new file mode 100644 index 0000000..e5fe465 --- /dev/null +++ b/fawnds/hash_table_cuckoo.cc @@ -0,0 +1,637 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ + +#include "hash_table_cuckoo.h" +#include "hash_functions.h" +#include "hashutil.h" +#include "configuration.h" +#include "print.h" +#include "file_io.h" +#include "debug.h" +#include "file_store.h" + +#include +#include +#include + +namespace fawn { + + HashTableCuckoo::HashTableCuckoo() + :hash_table_(NULL), fpf_table_(NULL) + { + DPRINTF(2, "%d-%d Cuckoo hash table\n", NUMHASH, ASSOCIATIVITY); + } + + HashTableCuckoo::~HashTableCuckoo() + { + if (hash_table_ || fpf_table_) + Close(); + } + + FawnDS_Return + HashTableCuckoo::Create() + { + DPRINTF(2, "HashTableCuckoo::Create()\n"); + + if (hash_table_ || fpf_table_) + Close(); + + string hts_string = config_->GetStringValue("child::hash-table-size"); + int hts_int = atoi(hts_string.c_str()); + if (hts_int <= 0) { + return ERROR; + } + uint64_t table_size = (uint64_t)hts_int; + +#ifdef SPLIT_HASHTABLE + max_index_ = (1 << KEYFRAGBITS) * NUMHASH; +#else + max_index_ = (1 << KEYFRAGBITS); +#endif + + if (max_index_ * ASSOCIATIVITY > table_size) + max_index_ = table_size / ASSOCIATIVITY; + + max_entries_ = ASSOCIATIVITY * max_index_; + current_entries_ = 0; + + DPRINTF(2, "HashTableCuckoo::Create(): given table_size=%llu\n", static_cast(max_entries_)); + + + DPRINTF(2, "CreateFawnDS table information:\n" + "\t KEYFRAGBITS: %ld\n" + "\t hashtablesize: %ld\n" + "\t num_entries: %ld\n" + "\t Maximum number of entries: %ld\n", + KEYFRAGBITS, max_index_, current_entires, max_entries_); + + if (config_->ExistsNode("child::use-offset") != 0 || atoi(config_->GetStringValue("child::use-offset").c_str()) != 0) { + hash_table_ = new TagValStoreEntry[max_index_]; + fpf_table_ = NULL; + + // zero out the buffer + memset(hash_table_, 0, sizeof(TagValStoreEntry) * max_index_); + + DPRINTF(2, "HashTableCuckoo::Create(): byte size=%zu\n", sizeof(TagValStoreEntry) * max_index_); + } + else { + hash_table_ = NULL; + fpf_table_ = new TagStoreEntry[max_index_]; + + // zero out the buffer + memset(fpf_table_, 0, sizeof(TagStoreEntry) * max_index_); + + DPRINTF(2, "HashTableCuckoo::Create(): byte size=%zu\n", sizeof(TagStoreEntry) * max_index_); + } + + return OK; + } + + FawnDS_Return + HashTableCuckoo::Open() + { + DPRINTF(2, "HashTableCuckoo::Open()\n"); + if (hash_table_ || fpf_table_) + Close(); + + if (Create() == OK) + if (ReadFromFile() == 0) + return OK; + + return ERROR; + + } + + FawnDS_Return + HashTableCuckoo::ConvertTo(FawnDS* new_store) const + { + HashTableCuckoo* new_cuckoo = dynamic_cast(new_store); + if (!new_cuckoo) { + fprintf(stderr, "HashTableCuckoo::ConvertTo(): Don't know how to convert\n"); + return UNSUPPORTED; + } + + if (max_index_ != new_cuckoo->max_index_) { + fprintf(stderr, "HashTableDefault::ConvertTo(): hash table size mismatch\n"); + return ERROR; + } + + if (hash_table_ == NULL && new_cuckoo->hash_table_ != NULL) { + DPRINTF(2, "HashTableCuckoo::ConvertTo(): insufficient information for conversion\n"); + return ERROR; + } + + DPRINTF(2, "HashTableCuckoo::ConvertTo(): converting to HashTableCuckoo\n"); + + if (hash_table_ != NULL && new_cuckoo->hash_table_ != NULL) + memcpy(new_cuckoo->hash_table_, hash_table_, sizeof(TagValStoreEntry) * max_index_); + else if (hash_table_ == NULL && new_cuckoo->hash_table_ == NULL) + memcpy(new_cuckoo->fpf_table_, fpf_table_, sizeof(TagStoreEntry) * max_index_); + else { + // copy bytes from h + for (size_t i = 0; i < max_index_; i++) { + memcpy(new_cuckoo->fpf_table_[i].tag_vector, hash_table_[i].tag_vector, sizeof(TagStoreEntry)); + } + } + new_cuckoo->current_entries_ = current_entries_; + return OK; + } + + FawnDS_Return + HashTableCuckoo::Flush() + { + DPRINTF(2, "HashTableCuckoo::Flush()\n"); + if (WriteToFile()) + return ERROR; + else + return OK; + + } + + FawnDS_Return + HashTableCuckoo::Close() + { + if (!hash_table_ && !fpf_table_) + return ERROR; + + Flush(); + + DPRINTF(2, "HashTableCuckoo::Close()\n"); + if (hash_table_) { + delete [] hash_table_; + DPRINTF(2, "HashTableCuckoo::Close(): HashTable deleted\n"); + } + if (fpf_table_) { + delete [] fpf_table_; + DPRINTF(2, "HashTableCuckoo::Close(): FpfTable deleted\n"); + } + hash_table_ = NULL; + fpf_table_ = NULL; + return OK; + } + + FawnDS_Return + HashTableCuckoo::Destroy() + { + std::string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + + if (unlink(filename.c_str())) { + perror("Could not delete file"); + return ERROR; + } + + return OK; + } + + FawnDS_Return + HashTableCuckoo::Status(const FawnDS_StatusType& type, Value& status) const + { + if (!hash_table_ && !fpf_table_) + return ERROR; + + std::ostringstream oss; + switch (type) { + case NUM_DATA: + oss << max_entries_; + break; + case NUM_ACTIVE_DATA: + oss << current_entries_; + break; + case CAPACITY: + oss << max_entries_; + break; + case MEMORY_USE: + if (hash_table_) + oss << sizeof(TagValStoreEntry) * max_index_; + else if (fpf_table_) + oss << sizeof(TagStoreEntry) * max_index_; + else + oss << 0; + break; + case DISK_USE: + if (hash_table_) + oss << sizeof(current_entries_) + sizeof(TagValStoreEntry) * max_index_; + else if (fpf_table_) + oss << sizeof(current_entries_) + sizeof(TagStoreEntry) * max_index_; + else + oss << sizeof(current_entries_); + break; + default: + return UNSUPPORTED; + } + status = NewValue(oss.str()); + return OK; + } + + FawnDS_Return + HashTableCuckoo::Put(const ConstValue& key, const ConstValue& data) + { +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "HashTableCuckoo::Put(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + DPRINTF(2, "HashTableCuckoo::Put(): data=%llu\n", static_cast(data.as_number())); + } +#endif + + DPRINTF(2, "HashTableCuckoo::Put(): key=\n"); + //print_payload((const u_char*)key.data(), key.size(), 4); + DPRINTF(2, "HashTableCuckoo::Put(): data=%llu\n", static_cast(data.as_number())); + + // for undo correctness checking + //uint32_t init_checksum = Hashes::h1(hash_table_, sizeof(TagValStoreEntry) * max_index_); + + uint32_t current_index, current_tag, current_val; + uint32_t victim_way, victim_index, victim_tag, victim_val; + uint32_t way; + + uint32_t undo_index[MAX_CUCKOO_COUNT]; + uint8_t undo_way[MAX_CUCKOO_COUNT]; + + uint32_t fn = rand() % NUMHASH; +#ifdef SPLIT_HASHTABLE + current_index = (keyfrag(key, fn) + fn * (1 << KEYFRAGBITS)) % max_index_; +#else + current_index = keyfrag(key, fn) % max_index_; +#endif + current_tag = keyfrag(key, (fn + 1) % NUMHASH) % max_index_; + + DPRINTF(2, "key: (index %d sig %d)\n", current_index, current_tag); + + current_val = static_cast (data.as_number()); + if ((way = freeslot(current_index)) < ASSOCIATIVITY) { + /* an empty slot @ (index, way) of hash table */ + store(current_index, way, current_tag | VALIDBITMASK, current_val); + current_entries_++; + return OK; + } + + fn = (fn + 1) % NUMHASH; +#ifdef SPLIT_HASHTABLE + current_index = (keyfrag(key, fn) + fn * (1 << KEYFRAGBITS)) % max_index_; +#else + current_index = keyfrag(key, fn) % max_index_; +#endif + current_tag = keyfrag(key, (fn + 1) % NUMHASH) % max_index_; + + for (uint32_t n = 0; n < MAX_CUCKOO_COUNT; n++) { + DPRINTF(2, "%d th try in loop!\n", n); + DPRINTF(2, "key: (index %d sig %d)\n", current_index, current_tag); + + if ((way = freeslot(current_index)) < ASSOCIATIVITY) { + /* an empty slot @ (current_index, current_way) of hash table */ + store(current_index, way, current_tag | VALIDBITMASK, current_val); + current_entries_++; + return OK; + } + + victim_index = current_index; + victim_way = rand() % ASSOCIATIVITY; + victim_tag = tag(victim_index, victim_way); + victim_val = val(victim_index, victim_way); + + undo_index[n] = victim_index; + undo_way[n] = victim_way; + + DPRINTF(2, "this bucket (%d) is full, take %d th as victim!\n", victim_index, victim_way); + //DPRINTF(2, "victim: (index %d sig %d)\n", victim_index, victim_tag); + store(victim_index, victim_way, current_tag | VALIDBITMASK, current_val); +#ifdef SPLIT_HASHTABLE + current_index = (victim_tag + (1 << KEYFRAGBITS)) % max_index_; + current_tag = victim_index % max_index_; +#else + current_index = victim_tag % max_index_; + current_tag = victim_index % max_index_; +#endif + current_val = victim_val; + } + + DPRINTF(2, "HashTableCuckoo::Put(): no more space to put new key -- undoing cuckooing\n"); + + // undo cuckooing + // restore the last state (find the previous victim_tag when calling last store()) +#ifdef SPLIT_HASHTABLE + current_tag = (current_index + (1 << KEYFRAGBITS)) % max_index_; +#else + current_tag = current_index % max_index_; +#endif + for (uint32_t n = 0; n < MAX_CUCKOO_COUNT; n++) { + victim_index = undo_index[MAX_CUCKOO_COUNT - 1 - n]; + victim_way = undo_way[MAX_CUCKOO_COUNT - 1 - n]; + victim_tag = tag(victim_index, victim_way); + victim_val = val(victim_index, victim_way); + + DPRINTF(2, "restoring victim (index %d sig %d)\n", victim_index, current_tag); + + store(victim_index, victim_way, current_tag | VALIDBITMASK, current_val); + +#ifdef SPLIT_HASHTABLE + current_tag = (victim_index + (1 << KEYFRAGBITS)) % max_index_; +#else + current_tag = victim_index % max_index_; +#endif + current_val = victim_val; + } + + assert(current_val == static_cast (data.as_number())); + + //uint32_t final_checksum = Hashes::h1(hash_table_, sizeof(TagValStoreEntry) * max_index_); + //DPRINTF(2, "HashTableCuckoo::Put(): initial checksum=%u, final checksum=%u\n", init_checksum, final_checksum); + + DPRINTF(2, "HashTableCuckoo::Put(): undoing done\n"); + + return INSUFFICIENT_SPACE; + } // HashTableCuckoo:Put() + + + FawnDS_ConstIterator + HashTableCuckoo::Enumerate() const + { + DPRINTF(2, "HashTableCuckoo::Enumerate() const\n"); + IteratorElem* elem = new IteratorElem(); + elem->fawnds = this; + + elem->current_index = static_cast(-1); + elem->current_way = ASSOCIATIVITY - 1; + + elem->Next(); + return FawnDS_ConstIterator(elem); + } + + FawnDS_Iterator + HashTableCuckoo::Enumerate() + { + DPRINTF(2, "HashTableCuckoo::Enumerate()\n"); + IteratorElem* elem = new IteratorElem(); + elem->fawnds = this; + + elem->current_index = static_cast(-1); + elem->current_way = ASSOCIATIVITY - 1; + + elem->Next(); + return FawnDS_Iterator(elem); + } + + FawnDS_ConstIterator + HashTableCuckoo::Find(const ConstValue& key) const + { + DPRINTF(2, "HashTableCuckoo::Find() const\n"); + IteratorElem* elem = new IteratorElem(); + elem->fawnds = this; + elem->key = key; + for (uint32_t i = 0 ; i < NUMHASH; i++) + elem->keyfrag[i] = keyfrag(key, i) % max_index_; + + elem->current_keyfrag_id = static_cast(-1); + elem->current_way = ASSOCIATIVITY - 1; + + elem->Next(); + return FawnDS_ConstIterator(elem); + } + + FawnDS_Iterator + HashTableCuckoo::Find(const ConstValue& key) + { + DPRINTF(2, "HashTableCuckoo::Find()\n"); + IteratorElem* elem = new IteratorElem(); + elem->fawnds = this; + elem->key = key; + for (uint32_t i = 0 ; i < NUMHASH; i++) + elem->keyfrag[i] = keyfrag(key, i) % max_index_; + + elem->current_keyfrag_id = static_cast(-1); + elem->current_way = ASSOCIATIVITY - 1; + + elem->Next(); + return FawnDS_Iterator(elem); + } + + FawnDS_IteratorElem* + HashTableCuckoo::IteratorElem::Clone() const + { + IteratorElem* elem = new IteratorElem(); + *elem = *this; + return elem; + } + + void + HashTableCuckoo::IteratorElem::Next() + { + DPRINTF(2, "HashTableCuckoo::IteratorEnum::Next\n"); + const HashTableCuckoo* table = static_cast(fawnds); + + bool cont = true; + + while (cont) { + current_way ++; + + if (current_way >= ASSOCIATIVITY ) { + // time to go to the next index + current_way = 0; + + if (key.size() != 0) { + // Find() + + current_keyfrag_id ++; + + if (current_keyfrag_id >= NUMHASH) { + state = END; + return; + } + +#ifdef SPLIT_HASHTABLE + current_index = (table->keyfrag(key, current_keyfrag_id) + current_keyfrag_id * (1 << KEYFRAGBITS)) % table->max_index_; +#else + current_index = (table->keyfrag(key, current_keyfrag_id)) % table->max_index_; +#endif + + } + else { + // Enumerate() + + current_index++; + + if (current_index >= table->max_index_) { + state = END; + return; + } + } + } + + // if (key.size() != 0) { + // // for Find(), current_index is derived from current_key_id + // } + + assert(current_index < table->max_index_); + + DPRINTF(2, "check (row %d, col %d) ... ", current_index, current_way); + + if (table->valid(current_index, current_way)) { + uint32_t tag = table->tag(current_index, current_way); + DPRINTF(2,"tag="); + //int_to_bit(tag, (KEYFRAGBITS )); + + if (key.size() != 0 && tag != keyfrag[(current_keyfrag_id + 1) % NUMHASH] ) { + // key mismatch + continue; + } + + } + else { + // unused space + continue; + } + + state = OK; + uint32_t v = table->val(current_index, current_way); + data = NewValue(&v); + break; + + + } +#ifdef DEBUG + + /* + if (debug_level & 2) { + DPRINTF(2, "HashTableCuckoo::IteratorElem::Next(): index %i with key %ld B @ %x data %ld B @ %x\n", current_index, key.size(), key.data(), data.size(), data.data()); + print_payload((const u_char*)key.data(), key.size()); + print_payload((const u_char*)data.data(), data.size()); + } + */ +#endif + + } + + FawnDS_Return + HashTableCuckoo::IteratorElem::Replace(const ConstValue& data) + { +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "HashTableCuckoo::IteratorElem::Replace(): index %zu, data=%llu\n", current_index, static_cast(data.as_number())); + } +#endif + HashTableCuckoo* table = static_cast(const_cast(fawnds)); + + uint32_t new_id = data.as_number(-1); + if (new_id == static_cast(-1)) { + DPRINTF(2, "HashTableCuckoo::IteratorElem::Replace(): could not parse data as ID\n"); + return INVALID_DATA; + } + + DPRINTF(2, "HashTableCuckoo::IteratorElem::Replace(): updated\n"); + table->hash_table_[current_index].val_vector[current_way] = new_id; + return OK; + } // IteratorElem::Replace + + int + HashTableCuckoo::WriteToFile() + { + uint64_t offset = 0; + void* table = NULL; + uint64_t length = 0; + + if (hash_table_) { + table = hash_table_; + length = sizeof(current_entries_) + sizeof(struct TagValStoreEntry) * max_index_; + } + else if (fpf_table_) { + table = fpf_table_; + length = sizeof(current_entries_) + sizeof(struct TagStoreEntry) * max_index_; + } + + std::string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + DPRINTF(2, "HashTableCuckoo::WriteToFile(): writing to %s\n", filename.c_str()); + int fd; + if ((fd = open(filename.c_str(), O_RDWR|O_CREAT|O_NOATIME, 0666)) == -1) + { + perror("Could not open file"); + return -1; + } + + if (ftruncate(fd, (off_t)length) == -1) { + fprintf(stderr, "Could not extend file to %"PRIu64" bytes: %s\n", + length, strerror(errno)); + } + lseek(fd, 0, SEEK_SET); + + if (fill_file_with_zeros(fd, length) < 0) { + fprintf(stderr, "Could not zero out hash_table file.\n"); + return -1; + } + + if (pwrite64(fd, ¤t_entries_, sizeof(current_entries_), offset) != sizeof(current_entries_)) { + perror("Error in writing"); + return -1; + } + offset += sizeof(current_entries_); + length -= sizeof(current_entries_); + + // write the hashtable to the file + ssize_t nwrite; + while ((nwrite = pwrite64(fd, table, length, offset)) > 0) { + length -= nwrite; + offset += nwrite; + if (nwrite < 0) { + perror("Error in writing"); + return -1; + } + } + fdatasync(fd); + if (close(fd) == -1) + { + perror("Could not close file"); + return -1; + } + return 0; + } + + // This assumes that the file was closed properly. + int + HashTableCuckoo::ReadFromFile() + { + std::string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + DPRINTF(2, "HashTableCuckoo::ReadFromFile(): reading from %s\n", filename.c_str()); + int fd; + if ((fd = open(filename.c_str(), O_RDWR|O_CREAT|O_NOATIME, 0666)) == -1) + { + perror("Could not open file"); + return -1; + } + + // TODO: support for fpf_table + + uint64_t offset = 0; + uint64_t length = sizeof(current_entries_) + sizeof(struct TagValStoreEntry) * max_index_; + + if (pread64(fd, ¤t_entries_, sizeof(current_entries_), offset) != sizeof(current_entries_)) { + perror("Error in reading hashtable from file\n"); + return -1; + } + offset += sizeof(current_entries_); + length -= sizeof(current_entries_); + + if (hash_table_ == NULL) { + perror("hash table is NULL\n"); + return -1; + } + ssize_t nread; + while ((nread = pread64(fd, hash_table_, length, offset)) > 0) { + length -= nread; + offset += nread; + if (nread < 0) { + perror("Error in reading hashtable from file\n"); + return -1; + } + } + if (close(fd) == -1) + { + perror("Could not close file"); + return -1; + } + return 0; + } + + +} // namespace fawn diff --git a/fawnds/hash_table_cuckoo.h b/fawnds/hash_table_cuckoo.h new file mode 100644 index 0000000..3acca31 --- /dev/null +++ b/fawnds/hash_table_cuckoo.h @@ -0,0 +1,220 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _HASH_TABLE_CUCKOO_H_ +#define _HASH_TABLE_CUCKOO_H_ + +#include "basic_types.h" +#include "fawnds.h" + +#include +#include "print.h" +#include "debug.h" + +#include + + +//#define SPLIT_HASHTABLE + +using namespace std; +namespace fawn { + +class Configuration; + +// configuration +// : "cuckoo" (fixed) +// : the ID of the file store +// : the file name prefix to store the hash table for startup +// : the number of entries that the hash table is expected to hold +// : 1 (default): use an explicit offset field of 4 bytes +// 0: do not use offsets; a location in the hash table becomes an offset + +class HashTableCuckoo : public FawnDS { + /* + * parameters for cuckoo + */ + static const uint32_t NUMHASH = 2; + static const uint32_t ASSOCIATIVITY = 4; + + static const uint32_t NUMVICTIM = 2; // size of victim table + static const uint32_t MAX_CUCKOO_COUNT = 128; + /* + * make sure KEYFRAGBITS + VALIDBITS <= 16 + */ + static const uint32_t KEYFRAGBITS = 15; + static const uint32_t KEYFRAGMASK = (1 << KEYFRAGBITS) - 1; + static const uint32_t VALIDBITMASK = KEYFRAGMASK + 1; + static const uint32_t KEYPRESENTMASK = VALIDBITMASK | KEYFRAGMASK; + +protected: + struct TagValStoreEntry { + char tag_vector[ASSOCIATIVITY * ( KEYFRAGBITS + 1) / 8]; + uint32_t val_vector[ASSOCIATIVITY]; + } __attribute__((__packed__)); + + struct TagStoreEntry { + char tag_vector[ASSOCIATIVITY * ( KEYFRAGBITS + 1) / 8]; + } __attribute__((__packed__)); + +public: + HashTableCuckoo(); + virtual ~HashTableCuckoo(); + + virtual FawnDS_Return Create(); + virtual FawnDS_Return Open(); + + virtual FawnDS_Return Flush(); + virtual FawnDS_Return Close(); + + virtual FawnDS_Return Destroy(); + + virtual FawnDS_Return Status(const FawnDS_StatusType& type, Value& status) const; + + virtual FawnDS_Return Put(const ConstValue& key, const ConstValue& data); + + virtual FawnDS_Return ConvertTo(FawnDS* new_store) const; + + + // not supported by now. + // virtual FawnDS_Return Get(const ConstValue& key, Value& data, size_t offset = 0, size_t len = -1) const; + //virtual FawnDS_Return Append(Value& key, const ConstValue& data); + //virtual FawnDS_Return Delete(const ConstValue& key); + //virtual FawnDS_Return Contains(const ConstValue& key) const; + //virtual FawnDS_Return Length(const ConstValue& key, size_t& len) const; + + virtual FawnDS_ConstIterator Enumerate() const; + virtual FawnDS_Iterator Enumerate(); + + virtual FawnDS_ConstIterator Find(const ConstValue& key) const; + virtual FawnDS_Iterator Find(const ConstValue& key); + + struct IteratorElem : public FawnDS_IteratorElem { + FawnDS_IteratorElem* Clone() const; + void Next(); + FawnDS_Return Replace(const ConstValue& data); + uint32_t keyfrag[NUMHASH]; + uint32_t current_keyfrag_id; + uint32_t current_index; + uint32_t current_way; + }; + +protected: + int WriteToFile(); + int ReadFromFile(); + +private: + TagValStoreEntry* hash_table_; + TagStoreEntry* fpf_table_; + + uint32_t max_index_; + uint32_t max_entries_; + uint32_t current_entries_; + + inline bool valid(uint32_t index, uint32_t way) const { + uint32_t pos = way * (KEYFRAGBITS + 1); + uint32_t offset = pos >> 3; + uint32_t tmp; + // this is not big-endian safe, and it accesses beyond the end of the table + /* + if (hash_table_) + tmp = *((uint32_t *) (hash_table_[index].tag_vector + offset)); + else + tmp = *((uint32_t *) (fpf_table_[index].tag_vector + offset)); + tmp = tmp >> (pos & 7); + */ + // specialized code + assert(KEYFRAGBITS == 15); + if (hash_table_) + tmp = *((uint16_t *) (hash_table_[index].tag_vector + offset)); + else + tmp = *((uint16_t *) (fpf_table_[index].tag_vector + offset)); + return (tmp & VALIDBITMASK); + } + + // read from tagvector in the hashtable + inline uint32_t tag(uint32_t index, uint32_t way) const { + uint32_t pos = way * (KEYFRAGBITS + 1); + uint32_t offset = pos >> 3; + uint32_t tmp; + // this is not big-endian safe, and it accesses beyond the end of the table + /* + if (hash_table_) + tmp = *((uint32_t *) (hash_table_[index].tag_vector + offset)); + else + tmp = *((uint32_t *) (fpf_table_[index].tag_vector + offset)); + tmp = tmp >> (pos & 7); + */ + // specialized code + assert(KEYFRAGBITS == 15); + if (hash_table_) + tmp = *((uint16_t *) (hash_table_[index].tag_vector + offset)); + else + tmp = *((uint16_t *) (fpf_table_[index].tag_vector + offset)); + return (tmp & KEYFRAGMASK); + } + + inline uint32_t val(uint32_t index, uint32_t way) const { + if (hash_table_) + return hash_table_[index].val_vector[way]; + else + return index * ASSOCIATIVITY + way; + } + + // store keyfrag + validbit to tagvector in the hashtable + void store(uint32_t index, uint32_t way, uint32_t keypresent, uint32_t id) { + assert(hash_table_); + assert(way < ASSOCIATIVITY); + uint32_t pos = way * (KEYFRAGBITS + 1); + uint32_t offset = pos >> 3; + uint32_t shift = pos & 7; + // this is not big-endian safe, and it accesses beyond the end of the table + /* + uint32_t *p= (uint32_t *) (hash_table_[index].tag_vector + offset); + uint32_t tmp = *p; + */ + // specialized code + assert(KEYFRAGBITS == 15); + uint16_t *p= (uint16_t *) (hash_table_[index].tag_vector + offset); + uint16_t tmp = *p; + tmp &= ~( KEYPRESENTMASK << shift); + *p = tmp | ( keypresent << shift);; + hash_table_[index].val_vector[way] = id; + } + + + + /* read the keyfragment from the key */ + inline uint32_t keyfrag(const ConstValue& key, uint32_t keyfrag_id) const { + assert(key.size() >= sizeof(uint32_t)); + // take the last 4 bytes + uint32_t tmp = *((uint32_t *) (key.data() + key.size() - 4)); + tmp = (tmp >> (keyfrag_id * KEYFRAGBITS)) & KEYFRAGMASK; + +#ifdef DEBUG + // DPRINTF(2, "\t\t key=\t"); + // print_payload((const u_char*) key.data(), key.size(), 4); + // DPRINTF(2, "\t\t%dth tag=\t", keyfrag_index); + // print_payload((const u_char*) &tmp, 2, 4); +#endif + return tmp; + } + + + /* check if the given row has a free slot, return its way */ + uint32_t freeslot(uint32_t index) { + uint32_t way; + for (way = 0; way < ASSOCIATIVITY; way++) { + DPRINTF(4, "check ... (%d, %d)\t", index, way); + if (!valid(index, way)) { + DPRINTF(4, "not used!\n"); + return way; + } + DPRINTF(4, "used...\n"); + } + return way; + } + + friend class IteratorElem; +}; + +} // namespace fawn + +#endif // #ifndef _HASH_TABLE_CUCKOO_H_ diff --git a/fawnds/hash_table_default.cc b/fawnds/hash_table_default.cc new file mode 100644 index 0000000..d86bc97 --- /dev/null +++ b/fawnds/hash_table_default.cc @@ -0,0 +1,603 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ + +#include "hash_table_default.h" +#include "hash_functions.h" +#include "hashutil.h" +#include "configuration.h" +#include "print.h" +#include "file_io.h" +#include "debug.h" + +#include +#include +#include + +namespace fawn { + + HashTableDefault::HashTableDefault() + : hash_table_(NULL), hash_table_size_(0), c_1_(1.), c_2_(0.) + { + } + + HashTableDefault::~HashTableDefault() + { + if (hash_table_) + Close(); + } + + FawnDS_Return + HashTableDefault::Create() + { + if (hash_table_) + Close(); + + // Calculate hashtable size based on number of entries and other hashtable parameters + // numObjects = num_entries - deleted_entries + // size = max(numObjects * 2, hash_table_size) + // this means hash_table_size is a lower bound on resizing. + + string hts_string = config_->GetStringValue("child::hash-table-size"); + int hts_int = atoi(hts_string.c_str()); + uint64_t hash_table_size = (uint64_t)hts_int; + + DPRINTF(2, "HashTableDefault::Create(): given hash_table_size=%llu\n", static_cast(hash_table_size)); + + uint64_t numObjects = hash_table_size * EXCESS_BUCKET_FACTOR; + uint64_t max_entries = HashUtil::FindNextHashSize(numObjects); + + DPRINTF(2, "CreateFawnDS table information:\n" + "\t hashtablesize: %"PRIu64"\n" + "\t num_entries: %i\n" + "\t Maximum number of entries: %"PRIu64"\n", + numObjects, 0, max_entries); + + hash_table_size_ = max_entries; + current_entries_ = 0; + + string pm_string = config_->GetStringValue("child::probing-mode"); + if (pm_string == "" || pm_string == "linear") { + c_1_ = 1.; + c_2_ = 0.; + } + else if (pm_string == "quadratic") { + c_1_ = 0.5; + c_2_ = 0.5; + } + else { + DPRINTF(2, "HashTableDefault::Create(): invalid probing mode %s\n", pm_string.c_str()); + return ERROR; + } + + DPRINTF(2, "HashTableDefault::Create(): actual hash_table_size=%llu\n", static_cast(hash_table_size)); + + hash_table_ = new HashEntry[hash_table_size_]; + // zero out the buffer + memset(hash_table_, 0, sizeof(HashEntry) * hash_table_size_); + DPRINTF(2, "HashTableDefault::Create(): byte size=%zu\n", sizeof(HashEntry) * hash_table_size_); + + cached_lookup_key_.resize(0); + cached_unused_index_ = -1; + + return OK; + } + + FawnDS_Return + HashTableDefault::Open() + { + if (hash_table_) + Close(); + + if (Create() == OK) + if (ReadFromFile() == 0) + return OK; + + cached_lookup_key_.resize(0); + cached_unused_index_ = -1; + + return ERROR; + } + + FawnDS_Return + HashTableDefault::ConvertTo(FawnDS* new_store) const + { + HashTableDefault* table = dynamic_cast(new_store); + if (!table) + return UNSUPPORTED; + + if (hash_table_size_ != table->hash_table_size_) { + fprintf(stderr, "HashTableDefault::ConvertTo(): hash table size mismatch\n"); + return ERROR; + } + + if (c_1_ != table->c_1_) { + fprintf(stderr, "HashTableDefault::ConvertTo(): hash table parameter mismatch\n"); + return ERROR; + } + + if (c_2_ != table->c_2_) { + fprintf(stderr, "HashTableDefault::ConvertTo(): hash table parameter mismatch\n"); + return ERROR; + } + + memcpy(table->hash_table_, hash_table_, sizeof(HashEntry) * hash_table_size_); + table->current_entries_ = current_entries_; + + return OK; + } + + + FawnDS_Return + HashTableDefault::Flush() + { + if (WriteToFile()) + return ERROR; + else + return OK; + } + + FawnDS_Return + HashTableDefault::Close() + { + Flush(); + + if (hash_table_) + delete [] hash_table_; + DPRINTF(2, "HashTableDefault::Close(): HashTable deleted\n"); + hash_table_ = NULL; + return OK; + } + + FawnDS_Return + HashTableDefault::Destroy() + { + std::string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + + if (unlink(filename.c_str())) { + perror("Could not delete file"); + return ERROR; + } + + return OK; + } + + FawnDS_Return + HashTableDefault::Status(const FawnDS_StatusType& type, Value& status) const + { + if (!hash_table_) + return ERROR; + + std::ostringstream oss; + switch (type) { + case NUM_DATA: + oss << hash_table_size_; + break; + case NUM_ACTIVE_DATA: + oss << current_entries_; + break; + case CAPACITY: + oss << hash_table_size_; + break; + case MEMORY_USE: + oss << sizeof(struct HashEntry) * hash_table_size_; + break; + case DISK_USE: + oss << sizeof(current_entries_) + sizeof(struct HashEntry) * hash_table_size_; + break; + default: + return UNSUPPORTED; + } + status = NewValue(oss.str()); + return OK; + } + + FawnDS_Return + HashTableDefault::Put(const ConstValue& key, const ConstValue& data) + { +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "HashTableDefault::Put(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + DPRINTF(2, "HashTableDefault::Put(): data=%llu\n", static_cast(data.as_number())); + } +#endif + + uint16_t new_vkey = keyfragment_from_key(key); + + uint32_t new_id = data.as_number(-1); + if (new_id == static_cast(-1)) { + DPRINTF(2, "HashTableDefault::Put(): could not parse data as ID\n"); + return INVALID_DATA; + } + + size_t unused_index; + { + tbb::spin_mutex::scoped_lock lock(cache_mutex_); + + if (cached_unused_index_ != static_cast(-1) && cached_lookup_key_ == key) { + DPRINTF(2, "HashTableDefault::Put(): using cached index for unused space\n"); + + unused_index = cached_unused_index_; + + cached_lookup_key_.resize(0); + cached_unused_index_ = -1; + } + else { + lock.release(); + + DPRINTF(2, "HashTableDefault::Put(): no cached index for unused space\n"); + FawnDS_Iterator it = Find(key); + + while (!it.IsEnd()) + ++it; + + const IteratorElem& elem = static_cast(*it); + + if (valid(elem.current_index)) { + DPRINTF(2, "HashTableDefault::Put(): no more space to put new key\n"); + return INSUFFICIENT_SPACE; + } + + unused_index = elem.current_index; + } + } + + hash_table_[unused_index].present_key = new_vkey | VALIDBITMASK; + hash_table_[unused_index].id = new_id; + + current_entries_++; + DPRINTF(2, "HashTableDefault::Put(): added\n"); + return OK; + } + + FawnDS_Return + HashTableDefault::Contains(const ConstValue& key) const + { +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "HashTableDefault::Contains(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + } +#endif + FawnDS_ConstIterator it = Find(key); + if (!it.IsEnd()) { + DPRINTF(2, "HashTableDefault::Contains(): found\n"); + return OK; + } + else { + DPRINTF(2, "HashTableDefault::Contains(): not found\n"); + return KEY_NOT_FOUND; + } + } + + FawnDS_Return + HashTableDefault::Length(const ConstValue& key, size_t& len) const + { +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "HashTableDefault::Length(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + } +#endif + len = sizeof(uint32_t); // HashEntry.id + DPRINTF(2, "HashTableDefault::Length(): len=%llu\n", static_cast(len)); + return Contains(key); + } + + FawnDS_Return + HashTableDefault::Get(const ConstValue& key, Value& data, size_t offset, size_t len) const + { +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "HashTableDefault::Get(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + } +#endif + // this is not very useful due to key collisions, but let's just have it + FawnDS_ConstIterator it = Find(key); + if (!it.IsEnd()) { + if (offset == 0 && len == static_cast(-1)) + data = it->data; + else { + if (offset > it->data.size()) { + data.resize(0); + return OK; + } + + if (offset + len > it->data.size()) + len = it->data.size() - offset; + + data = NewValue(it->data.data() + offset, len); + } +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "HashTableDefault::Get(): data=\n"); + print_payload((const u_char*)data.data(), data.size(), 4); + } +#endif + return OK; + } + else { + DPRINTF(2, "HashTableDefault::Get(): not found\n"); + return KEY_NOT_FOUND; + } + } + + FawnDS_ConstIterator + HashTableDefault::Enumerate() const + { + IteratorElem* elem = new IteratorElem(); + elem->fawnds = this; + elem->vkey = 0; + + elem->current_probe_count = 0; + elem->max_probe_count = hash_table_size_; + + elem->next_hash_fn_index = Hashes::HASH_COUNT; + + elem->first_index = elem->current_index = 0; + + elem->skip_unused = true; + + elem->Next(); + return FawnDS_ConstIterator(elem); + } + + FawnDS_Iterator + HashTableDefault::Enumerate() + { + IteratorElem* elem = new IteratorElem(); + elem->fawnds = this; + elem->vkey = 0; + + elem->current_probe_count = 0; + elem->max_probe_count = hash_table_size_; + + elem->next_hash_fn_index = Hashes::HASH_COUNT; + + elem->first_index = elem->current_index = 0; + + elem->skip_unused = true; + + elem->Next(); + return FawnDS_Iterator(elem); + } + + FawnDS_ConstIterator + HashTableDefault::Find(const ConstValue& key) const + { + IteratorElem* elem = new IteratorElem(); + elem->fawnds = this; + elem->key = key; + elem->vkey = keyfragment_from_key(key); + + elem->current_probe_count = 0; + elem->max_probe_count = 0; // invokes the selection of the next hash function + + elem->next_hash_fn_index = 0; + + elem->first_index = elem->current_index = 0; + + elem->skip_unused = false; + + elem->Next(); + return FawnDS_ConstIterator(elem); + } + + FawnDS_Iterator + HashTableDefault::Find(const ConstValue& key) + { + IteratorElem* elem = new IteratorElem(); + elem->fawnds = this; + elem->key = key; + elem->vkey = keyfragment_from_key(key); + + elem->current_probe_count = 0; + elem->max_probe_count = 0; // invokes the selection of the next hash function + + elem->next_hash_fn_index = 0; + + elem->first_index = elem->current_index = 0; + + elem->skip_unused = false; + + elem->Next(); + return FawnDS_Iterator(elem); + } + + FawnDS_IteratorElem* + HashTableDefault::IteratorElem::Clone() const + { + IteratorElem* elem = new IteratorElem(); + *elem = *this; + return elem; + } + + void + HashTableDefault::IteratorElem::Next() + { + const HashTableDefault* table = static_cast(fawnds); + + while (true) { + if (current_probe_count >= max_probe_count) { + if (next_hash_fn_index == Hashes::HASH_COUNT) { + state = END; + break; + } + current_probe_count = 0; + max_probe_count = PROBES_BEFORE_REHASH; + first_index = (*(Hashes::hashes[next_hash_fn_index++]))(key.data(), key.size()); + first_index &= table->hash_table_size_ - 1; + } + + current_probe_count++; + + double n_i = first_index + + table->c_1_ * current_probe_count + + table->c_2_ * current_probe_count * current_probe_count; + current_index = static_cast(n_i + 0.5); + current_index &= table->hash_table_size_ - 1; + + DPRINTF(2, "HashTableDefault::IteratorElem::Next(): index %zu with probe count %zu\n", current_index, current_probe_count); + + if (table->valid(current_index)) { + if (key.size() == 0 || table->verifykey(current_index) == vkey) { + data = RefValue(&table->hash_table_[current_index].id); + state = OK; + break; + } + else { + // keyfrag mismatch + continue; + } + } + else { + if (skip_unused) + continue; + + { + tbb::spin_mutex::scoped_lock lock(table->cache_mutex_); + table->cached_lookup_key_ = key; + table->cached_unused_index_ = current_index; + } + + state = END; + break; + } + } + +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "HashTableDefault::IteratorElem::Next(): now at index %zu\n", current_index); + if (state == OK) { + DPRINTF(2, "HashTableDefault::IteratorElem::Next(): key=\n"); + print_payload((const u_char*)key.data(), key.size(), 4); + DPRINTF(2, "HashTableDefault::IteratorElem::Next(): data=%llu\n", static_cast(data.as_number())); + } + } +#endif + } + + FawnDS_Return + HashTableDefault::IteratorElem::Replace(const ConstValue& data) + { +#ifdef DEBUG + if (debug_level & 2) { + DPRINTF(2, "HashTableDefault::IteratorElem::Replace(): index %zu, data=%llu\n", current_index, static_cast(data.as_number())); + } +#endif + + HashTableDefault* table = static_cast(const_cast(fawnds)); + + uint32_t new_id = data.as_number(-1); + if (new_id == static_cast(-1)) { + DPRINTF(2, "HashTableDefault::IteratorElem::Replace(): could not parse data as ID\n"); + return INVALID_DATA; + } + + DPRINTF(2, "HashTableDefault::IteratorElem::Replace(): updated\n"); + table->hash_table_[current_index].id = new_id; + return OK; + } + + int + HashTableDefault::WriteToFile() + { + uint64_t offset = 0; + uint64_t length = sizeof(current_entries_) + sizeof(struct HashEntry) * hash_table_size_; + + std::string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + + DPRINTF(2, "HashTableDefault::WriteToFile(): writing to %s\n", filename.c_str()); + int fd; + if ((fd = open(filename.c_str(), O_RDWR|O_CREAT|O_NOATIME, 0666)) == -1) + { + perror("Could not open file"); + return -1; + } + + if (ftruncate(fd, (off_t)length) == -1) { + fprintf(stderr, "Could not extend file to %"PRIu64" bytes: %s\n", + length, strerror(errno)); + } + lseek(fd, 0, SEEK_SET); + + if (fill_file_with_zeros(fd, length) < 0) { + fprintf(stderr, "Could not zero out hash_table file.\n"); + return -1; + } + + if (pwrite64(fd, ¤t_entries_, sizeof(current_entries_), offset) != sizeof(current_entries_)) { + perror("Error in writing"); + return -1; + } + offset += sizeof(current_entries_); + length -= sizeof(current_entries_); + + // write the hashtable to the file + ssize_t nwrite; + while ((nwrite = pwrite64(fd, hash_table_, length, offset)) > 0) { + length -= nwrite; + offset += nwrite; + if (nwrite < 0) { + perror("Error in writing"); + return -1; + } + } + fdatasync(fd); + if (close(fd) == -1) + { + perror("Could not close file"); + return -1; + } + return 0; + } + + // This assumes that the file was closed properly. + int + HashTableDefault::ReadFromFile() + { + std::string filename = config_->GetStringValue("child::file") + "_"; + filename += config_->GetStringValue("child::id"); + + DPRINTF(2, "HashTableDefault::ReadFromFile(): reading from %s\n", filename.c_str()); + int fd; + if ((fd = open(filename.c_str(), O_RDWR|O_CREAT|O_NOATIME, 0666)) == -1) + { + perror("Could not open file"); + return -1; + } + + uint64_t offset = 0; + uint64_t length = sizeof(current_entries_) + sizeof(struct HashEntry) * hash_table_size_; + + if (pread64(fd, ¤t_entries_, sizeof(current_entries_), offset) != sizeof(current_entries_)) { + perror("Error in reading hashtable from file\n"); + return -1; + } + offset += sizeof(current_entries_); + length -= sizeof(current_entries_); + + if (hash_table_ == NULL) { + perror("hash table is NULL\n"); + return -1; + } + ssize_t nread; + while ((nread = pread64(fd, hash_table_, length, offset)) > 0) { + length -= nread; + offset += nread; + if (nread < 0) { + perror("Error in reading hashtable from file\n"); + return -1; + } + } + if (close(fd) == -1) + { + perror("Could not close file"); + return -1; + } + return 0; + } + +} // namespace fawn diff --git a/fawnds/hash_table_default.h b/fawnds/hash_table_default.h new file mode 100644 index 0000000..78c47ec --- /dev/null +++ b/fawnds/hash_table_default.h @@ -0,0 +1,140 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _HASH_TABLE_DEFAULT_H_ +#define _HASH_TABLE_DEFAULT_H_ + +#include "basic_types.h" +#include "fawnds.h" +#include + +namespace fawn { + +class Configuration; + +// configuration +// : "default" (fixed) +// : the ID of the file store +// : the file name prefix to store the hash table for startup +// : the number of entries that the hash table is expected to hold +// : "linear" (default) for linear probing +// "quadratic" for quadratic probing + +class HashTableDefault : public FawnDS { + // Incrementing keyfragbits above 15 requires + // more modifications to code (e.g. hashkey is 16 bits in (Insert()) + static const uint32_t KEYFRAGBITS = 15; + static const uint32_t KEYFRAGMASK = (1 << KEYFRAGBITS) - 1; + static const uint32_t INDEXBITS = 16; + static const uint32_t INDEXMASK = (1 << INDEXBITS) - 1; + static const uint32_t VALIDBITMASK = KEYFRAGMASK+1; + + static const double EXCESS_BUCKET_FACTOR = 1.1; + static const double MAX_DELETED_RATIO = 0.8; + static const double MAX_LOAD_FACTOR = 0.9; + + static const double PROBES_BEFORE_REHASH = 8; + //static const double PROBES_BEFORE_REHASH = 16; + +protected: + /* + Hash Entry Format + D = Is slot deleted: 1 means deleted, 0 means not deleted. Needed for lazy deletion + V = Is slot empty: 0 means empty, 1 means taken + K = Key fragment + O = Offset bits + ________________________________________________ + |DVKKKKKKKKKKKKKKOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO| + ¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯ + */ + struct HashEntry { + uint16_t present_key; + uint32_t id; + } __attribute__((__packed__)); + + +public: + HashTableDefault(); + virtual ~HashTableDefault(); + + virtual FawnDS_Return Create(); + virtual FawnDS_Return Open(); + + virtual FawnDS_Return ConvertTo(FawnDS* new_store) const; + + virtual FawnDS_Return Flush(); + virtual FawnDS_Return Close(); + + virtual FawnDS_Return Destroy(); + + virtual FawnDS_Return Status(const FawnDS_StatusType& type, Value& status) const; + + virtual FawnDS_Return Put(const ConstValue& key, const ConstValue& data); + //virtual FawnDS_Return Append(Value& key, const ConstValue& data); + + //virtual FawnDS_Return Delete(const ConstValue& key); + + virtual FawnDS_Return Contains(const ConstValue& key) const; + virtual FawnDS_Return Length(const ConstValue& key, size_t& len) const; + virtual FawnDS_Return Get(const ConstValue& key, Value& data, size_t offset = 0, size_t len = -1) const; + + virtual FawnDS_ConstIterator Enumerate() const; + virtual FawnDS_Iterator Enumerate(); + + virtual FawnDS_ConstIterator Find(const ConstValue& key) const; + virtual FawnDS_Iterator Find(const ConstValue& key); + + struct IteratorElem : public FawnDS_IteratorElem { + FawnDS_IteratorElem* Clone() const; + void Next(); + FawnDS_Return Replace(const ConstValue& data); + + uint16_t vkey; + + size_t current_probe_count; + size_t max_probe_count; + + size_t next_hash_fn_index; + + size_t first_index; + size_t current_index; + + bool skip_unused; + }; + +protected: + int WriteToFile(); + int ReadFromFile(); + +private: + HashEntry* hash_table_; + size_t hash_table_size_; + size_t current_entries_; + + // probe increment function coefficients + double c_1_; + double c_2_; + + mutable tbb::spin_mutex cache_mutex_; + mutable Value cached_lookup_key_; + mutable size_t cached_unused_index_; + + inline bool valid(int32_t index) const { + return (hash_table_[index].present_key & VALIDBITMASK); + } + + inline uint16_t verifykey(int32_t index) const { + return (hash_table_[index].present_key & KEYFRAGMASK); + } + + inline uint16_t keyfragment_from_key(const ConstValue& key) const { + if (key.size() < sizeof(uint16_t)) { + return (uint16_t) (key.data()[0] & KEYFRAGMASK); + } + return (uint16_t) (((key.data()[key.size()-2]<<8) + (key.data()[key.size()-1])) & KEYFRAGMASK); + } + + friend class IteratorElem; +}; + +} // namespace fawn + +#endif // #ifndef _HASH_TABLE_DEFAULT_H_ diff --git a/fawnds/rate_limiter.cc b/fawnds/rate_limiter.cc new file mode 100644 index 0000000..332f140 --- /dev/null +++ b/fawnds/rate_limiter.cc @@ -0,0 +1,110 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "rate_limiter.h" +#include "debug.h" +#include +#include +#include + +namespace fawn { + + RateLimiter::RateLimiter(int64_t initial_tokens, int64_t max_tokens, int64_t new_tokens_per_interval, int64_t ns_per_interval, int64_t min_retry_interval_ns) + : max_tokens_(max_tokens), new_tokens_per_interval_(new_tokens_per_interval), ns_per_interval_(ns_per_interval), min_retry_interval_ns_(min_retry_interval_ns) + { + tokens_ = initial_tokens; + +#ifndef __APPLE__ + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts)) + perror("Error while calling clock_gettime()"); + last_time_ = static_cast(ts.tv_sec) * 1000000000Lu + static_cast(ts.tv_nsec); +#else + struct timeval tv; + if (gettimeofday(&tv, NULL)) + perror("Error while calling gettimeofday()"); + last_time_ = static_cast(tv.tv_sec) * 1000000000Lu + static_cast(tv.tv_usec) * 1000Lu; +#endif + } + + void + RateLimiter::remove_tokens(int64_t v) + { + while (true) { + update_tokens(); + + int64_t extra_tokens_required = v - tokens_; + + if (extra_tokens_required <= 0) { + // it is OK to make tokens_ negative + tokens_ -= v; + return; + } + + int64_t minimum_ns = extra_tokens_required * ns_per_interval_ / new_tokens_per_interval_; + if (minimum_ns < min_retry_interval_ns_) + minimum_ns = min_retry_interval_ns_; + + struct timespec ts; + ts.tv_sec = minimum_ns / 1000000000L; + ts.tv_nsec = minimum_ns % 1000000000L; + //fprintf(stderr, "%Lu %Lu\n", ts.tv_sec, ts.tv_nsec); + nanosleep(&ts, NULL); + } + } + + bool + RateLimiter::try_remove_tokens(int64_t v) + { + update_tokens(); + + int64_t extra_tokens_required = v - tokens_; + + if (extra_tokens_required <= 0) { + // it is OK to make tokens_ negative + tokens_ -= v; + return true; + } + else + return false; + } + + void + RateLimiter::update_tokens() + { + // TODO: handle time overflow in timespec/timeval +#ifndef __APPLE__ + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts)) + perror("Error while calling clock_gettime()"); + int64_t current_time = static_cast(ts.tv_sec) * 1000000000Lu + static_cast(ts.tv_nsec); +#else + struct timeval tv; + if (gettimeofday(&tv, NULL)) + perror("Error while calling gettimeofday()"); + int64_t current_time = static_cast(tv.tv_sec) * 1000000000Lu + static_cast(tv.tv_usec) * 1000Lu; +#endif + + int64_t new_intervals = (current_time - last_time_) / ns_per_interval_; + if (new_intervals < 0) { + // someone else consumed too much intervals + return; + } + + // consume new interval times + last_time_ += new_intervals * ns_per_interval_; + + int64_t current_tokens = tokens_; + + int64_t new_tokens = new_intervals * new_tokens_per_interval_; + if (new_tokens + current_tokens > max_tokens_) { + // trying to avoid exceeding maximum # of tokens + new_tokens = max_tokens_ - current_tokens; + if (new_tokens < 0) { + // the token number is maxed out + return; + } + } + + tokens_ += new_tokens; + } + +} // namespace fawn diff --git a/fawnds/rate_limiter.h b/fawnds/rate_limiter.h new file mode 100644 index 0000000..64adbf9 --- /dev/null +++ b/fawnds/rate_limiter.h @@ -0,0 +1,52 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _RATE_LIMITER_H_ +#define _RATE_LIMITER_H_ + +#include + +namespace fawn { + + // a high-performance high-precision rate limiter based on token bucket + + class RateLimiter { + public: + RateLimiter(int64_t initial_tokens, int64_t max_tokens, int64_t new_tokens_per_interval, int64_t ns_per_interval, int64_t min_retry_interval_ns = 1000L); + + void remove_tokens(int64_t v); + bool try_remove_tokens(int64_t v); + + void remove_tokens_nowait(int64_t v) { tokens_ -= v; } + + void add_tokens(int64_t v) { tokens_ += v; } + + int64_t tokens() const { return tokens_; } + void set_tokens(int64_t v) { tokens_ = v; } + + const int64_t& max_tokens() const { return max_tokens_; } + void set_max_tokens(int64_t v) { max_tokens_ = v; } + + const int64_t& new_tokens_per_interval() const { return new_tokens_per_interval_; } + void set_new_tokens_per_interval(int64_t v) { new_tokens_per_interval_ = v; } + + const int64_t& ns_per_interval() const { return ns_per_interval_; } + void set_ns_per_interval(int64_t v) { ns_per_interval_ = v; } + + const int64_t& min_retry_interval_ns() const { return min_retry_interval_ns_; } + void set_min_retry_interval_ns(int64_t v) { min_retry_interval_ns_ = v; } + + protected: + void update_tokens(); + + private: + tbb::atomic tokens_; + int64_t max_tokens_; + int64_t new_tokens_per_interval_; + int64_t ns_per_interval_; + int64_t min_retry_interval_ns_; + + tbb::atomic last_time_; + }; + +} // namespace fawn + +#endif // #ifndef _RATE_LIMITER_H_ diff --git a/fawnds/sorter.cc b/fawnds/sorter.cc new file mode 100644 index 0000000..dc7ad57 --- /dev/null +++ b/fawnds/sorter.cc @@ -0,0 +1,268 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "sorter.h" +#include "debug.h" +#include "configuration.h" +#ifndef HAVE_LIBNSORT +#include +#endif +#include + +namespace fawn { + + Sorter::Sorter() + : open_(false) + { + } + + Sorter::~Sorter() + { + if (open_) + Close(); + } + + FawnDS_Return + Sorter::Create() + { + if (open_) + return ERROR; + + if (config_->ExistsNode("child::key-len") == 0) + key_len_ = atoi(config_->GetStringValue("child::key-len").c_str()); + else + key_len_ = 0; + + if (config_->ExistsNode("child::data-len") == 0) + data_len_ = atoi(config_->GetStringValue("child::data-len").c_str()); + else + data_len_ = 0; + + if (config_->ExistsNode("child::temp-file") == 0) + temp_file_ = config_->GetStringValue("child::temp-file"); + else + temp_file_ = "/tmp"; + +#ifdef HAVE_LIBNSORT + char buf[1024]; + + if (key_len_ != 0 && data_len_ != 0) { + const char* nsort_definition = + " -format=size: %zu" + " -field=Key, char, offset: 0, size: %zu" + " -key=Key" + " -temp_file=%s"; + + snprintf(buf, sizeof(buf), nsort_definition, key_len_ + data_len_, key_len_, temp_file_.c_str()); + } + else { + // currently not supported + return ERROR; + } + + nsort_msg_t ret = nsort_define(buf, 0, NULL, &nsort_ctx_); + if (ret != NSORT_SUCCESS) { + fprintf(stderr, "Nsort error: %s\n", nsort_message(&nsort_ctx_)); + return ERROR; + } + + nsort_buf_.resize(key_len_ + data_len_); +#endif + + open_ = true; + input_ended_ = false; + + return OK; + } + +#ifndef HAVE_LIBNSORT + struct _ordering { + const std::vector& key_array; + + _ordering(const std::vector& key_array) : key_array(key_array) {} + + bool operator()(const size_t& i, const size_t& j) { + // sort key = (entry key, index) + int cmp = key_array[i].compare(key_array[j]); + if (cmp != 0) + return cmp < 0; + else + return i < j; // stable sorting + }; + }; +#endif + + FawnDS_Return + Sorter::Flush() + { + if (!open_) + return ERROR; + if (input_ended_) + return ERROR; + + input_ended_ = true; + +#ifndef HAVE_LIBNSORT + refs_.reserve(key_array_.size()); + for (size_t i = 0; i < key_array_.size(); i++) + refs_.push_back(i); + + // uses unstable sorting with stable sort key comparison + // because std::sort() is often more efficient than std::stable_sort() + std::sort(refs_.begin(), refs_.end(), _ordering(key_array_)); + + refs_it_ = refs_.begin(); +#else + nsort_msg_t ret = nsort_release_end(&nsort_ctx_); + if (ret != NSORT_SUCCESS) + fprintf(stderr, "Nsort error: %s\n", nsort_message(&nsort_ctx_)); +#endif + + return OK; + } + + FawnDS_Return + Sorter::Close() + { + if (!open_) + return ERROR; + +#ifndef HAVE_LIBNSORT + key_array_.clear(); + data_array_.clear(); + refs_.clear(); +#else + nsort_end(&nsort_ctx_); + nsort_buf_.resize(0); +#endif + + open_ = false; + return OK; + } + + FawnDS_Return + Sorter::Put(const ConstValue& key, const ConstValue& data) + { + if (!open_) + return ERROR; + if (input_ended_) + return ERROR; + + if (key.size() != key_len_) + return INVALID_KEY; + if (data.size() != data_len_) + return INVALID_DATA; + +#ifndef HAVE_LIBNSORT + NewValue copied_key(key.data(), key.size()); + NewValue copied_value(data.data(), data.size()); + key_array_.push_back(copied_key); + data_array_.push_back(copied_value); +#else + memcpy(nsort_buf_.data(), key.data(), key_len_); + memcpy(nsort_buf_.data() + key_len_, data.data(), data_len_); + + nsort_msg_t ret = nsort_release_recs(nsort_buf_.data(), key_len_ + data_len_, &nsort_ctx_); + if (ret != NSORT_SUCCESS) + fprintf(stderr, "Nsort error: %s\n", nsort_message(&nsort_ctx_)); +#endif + + return OK; + } + + FawnDS_ConstIterator + Sorter::Enumerate() const + { + if (!open_) + return FawnDS_ConstIterator(); + if (!input_ended_) + return FawnDS_ConstIterator(); + + IteratorElem* elem = new IteratorElem(this); + elem->state = OK; + elem->Increment(true); + + return FawnDS_ConstIterator(elem); + } + + FawnDS_Iterator + Sorter::Enumerate() + { + if (!open_) + return FawnDS_Iterator(); + if (!input_ended_) + return FawnDS_Iterator(); + + IteratorElem* elem = new IteratorElem(this); + elem->state = OK; + elem->Increment(true); + + return FawnDS_Iterator(elem); + } + + Sorter::IteratorElem::IteratorElem(const Sorter* fawnds) + { + this->fawnds = fawnds; + } + + FawnDS_IteratorElem* + Sorter::IteratorElem::Clone() const + { + IteratorElem* elem = new IteratorElem(static_cast(fawnds)); + *elem = *this; + return elem; + } + + void + Sorter::IteratorElem::Next() + { + Increment(false); + } + + void + Sorter::IteratorElem::Increment(bool initial) + { + if (state == END) + return; + + const Sorter* sorter = static_cast(fawnds); + +#ifndef HAVE_LIBNSORT + + std::vector::const_iterator& refs_it = sorter->refs_it_; + + if (!initial) + ++refs_it; + + if (refs_it == sorter->refs_.end()) { + state = END; + return; + } + + state = OK; + key = sorter->key_array_[*refs_it]; + data = sorter->data_array_[*refs_it]; + +#else + + (void)initial; + + size_t size = sorter->key_len_ + sorter->data_len_; + nsort_msg_t ret = nsort_return_recs(sorter->nsort_buf_.data(), &size, &sorter->nsort_ctx_); + if (ret == NSORT_SUCCESS) { + assert(size == sorter->key_len_ + sorter->data_len_); + key = NewValue(sorter->nsort_buf_.data(), sorter->key_len_); + data = NewValue(sorter->nsort_buf_.data() + sorter->key_len_, sorter->data_len_); + state = OK; + } + else if (ret == NSORT_END_OF_OUTPUT) + state = END; + else { + fprintf(stderr, "Nsort error: %s\n", nsort_message(&sorter->nsort_ctx_)); + assert(false); + state = END; + } + +#endif + } + +} // namespace fawn + diff --git a/fawnds/sorter.h b/fawnds/sorter.h new file mode 100644 index 0000000..b131273 --- /dev/null +++ b/fawnds/sorter.h @@ -0,0 +1,81 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _SORTER_H_ +#define _SORTER_H_ + +#include "fawnds.h" + +#include "config.h" +#ifndef HAVE_LIBNSORT +#include +#else +#include +#endif + +namespace fawn { + + // configuration + // : "sorter" (fixed) + // : the length of keys -- zero for variable-length keys (default), a positive integer for fixed-length keys + // : the length of data -- zero for variable-length data (default), a positive integer for fixed-length data + // : the path of the temporary files/directory. "/tmp" is the default. + + class Sorter : public FawnDS { + public: + Sorter(); + virtual ~Sorter(); + + virtual FawnDS_Return Create(); + //virtual FawnDS_Return Open(); + + virtual FawnDS_Return Flush(); + virtual FawnDS_Return Close(); + + //virtual FawnDS_Return Destroy(); + + virtual FawnDS_Return Put(const ConstValue& key, const ConstValue& data); + //virtual FawnDS_Return Append(Value& key, const ConstValue& data); + + //virtual FawnDS_Return Delete(const ConstValue& key); + + //virtual FawnDS_Return Contains(const ConstValue& key) const; + //virtual FawnDS_Return Length(const ConstValue& key, size_t& len) const; + //virtual FawnDS_Return Get(const ConstValue& key, Value& data, size_t offset = 0, size_t len = -1) const; + + virtual FawnDS_ConstIterator Enumerate() const; + virtual FawnDS_Iterator Enumerate(); + + //virtual FawnDS_ConstIterator Find(const ConstValue& key) const; + //virtual FawnDS_Iterator Find(const ConstValue& key); + + struct IteratorElem : public FawnDS_IteratorElem { + IteratorElem(const Sorter* fawnds); + + FawnDS_IteratorElem* Clone() const; + void Next(); + + void Increment(bool initial); + }; + + protected: + bool open_; + bool input_ended_; + + size_t key_len_; + size_t data_len_; + + std::string temp_file_; + +#ifndef HAVE_LIBNSORT + std::vector key_array_; + std::vector data_array_; + std::vector refs_; + mutable std::vector::const_iterator refs_it_; +#else + mutable nsort_t nsort_ctx_; + mutable Value nsort_buf_; +#endif + }; + +} // namespace fawn + +#endif // #ifndef _SORTER_H_ diff --git a/fawnds/task.cc b/fawnds/task.cc new file mode 100644 index 0000000..07f787d --- /dev/null +++ b/fawnds/task.cc @@ -0,0 +1,180 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "task.h" +#include "debug.h" +#include + +#include +#include + +#ifdef __linux__ + +#include +#include + +extern "C" { +pid_t gettid() +{ + return (pid_t)syscall(SYS_gettid); +} +} + +extern "C" { +// from linux/ioprio.h + +#define IOPRIO_BITS (16) +#define IOPRIO_CLASS_SHIFT (13) +#define IOPRIO_PRIO_MASK ((1UL << IOPRIO_CLASS_SHIFT) - 1) + +#define IOPRIO_PRIO_CLASS(mask) ((mask) >> IOPRIO_CLASS_SHIFT) +#define IOPRIO_PRIO_DATA(mask) ((mask) & IOPRIO_PRIO_MASK) +#define IOPRIO_PRIO_VALUE(class, data) (((class) << IOPRIO_CLASS_SHIFT) | data) + +#define ioprio_valid(mask) (IOPRIO_PRIO_CLASS((mask)) != IOPRIO_CLASS_NONE) + +enum { + IOPRIO_CLASS_NONE, + IOPRIO_CLASS_RT, + IOPRIO_CLASS_BE, + IOPRIO_CLASS_IDLE, +}; + +enum { + IOPRIO_WHO_PROCESS = 1, + IOPRIO_WHO_PGRP, + IOPRIO_WHO_USER, +}; + +static inline int ioprio_set(int which, int who, int ioprio) +{ + return syscall(__NR_ioprio_set, which, who, ioprio); +} + +static inline int ioprio_get(int which, int who) +{ + return syscall(__NR_ioprio_get, which, who); +} + +} +#endif + + +namespace fawn { + TaskScheduler::TaskScheduler(size_t num_workers, size_t queue_capacity, cpu_priority_t cpu_priority, io_priority_t io_priority) + : cpu_priority_(cpu_priority), io_priority_(io_priority) + { + assert(num_workers > 0); + + shared_queue_.set_capacity(queue_capacity); + + for (size_t i = 0; i < num_workers; i++) { + Worker* worker = new Worker(); + worker->owner = this; + worker->seen_join = false; + worker->queue = &shared_queue_; + //worker->queue.set_capacity(queue_capacity); + + if (pthread_create(&worker->tid, NULL, Worker::thread_main, worker) != 0) { + DPRINTF(2, "TaskScheduler::TaskScheduler(): failed to create worker\n"); + break; + } + + workers_.push_back(worker); + } + + //next_worker_ = 0; + } + + TaskScheduler::~TaskScheduler() + { + if (workers_.size()) + join(); + } + + void + TaskScheduler::enqueue_task(Task* t) + { + //workers_[next_worker_++ % workers_.size()]->queue.push(t); + shared_queue_.push(t); + } + + void + TaskScheduler::join() + { + for (size_t i = 0; i < workers_.size(); i++) { + JoinTask* t = new JoinTask(); + //workers_[i]->queue.push(t); + shared_queue_.push(t); + } + + for (size_t i = 0; i < workers_.size(); i++) { + pthread_join(workers_[i]->tid, NULL); + delete workers_[i]; + workers_[i] = NULL; + } + + workers_.clear(); + } + + void* + TaskScheduler::Worker::thread_main(void* arg) + { + Worker* this_worker = reinterpret_cast(arg); + +#ifdef __linux__ + pid_t tid = gettid(); + + { + if (this_worker->owner->cpu_priority_ == CPU_LOW) { + int new_nice = nice(1); + //fprintf(stderr, "set nice() = %d\n", new_nice); + } + } + + { + // from Documentation/block/ioprio.txt + int ioprio = 4; + int ioprio_class = IOPRIO_CLASS_BE; + + switch (this_worker->owner->io_priority_) { + case IO_REAL_TIME: + ioprio_class = IOPRIO_CLASS_RT; + break; + case IO_BEST_EFFORT: + ioprio_class = IOPRIO_CLASS_BE; + break; + case IO_IDLE: + ioprio_class = IOPRIO_CLASS_IDLE; + ioprio = 7; + break; + default: + assert(false); + break; + } + + int ret = ioprio_set(IOPRIO_WHO_PROCESS, tid, IOPRIO_PRIO_VALUE(ioprio_class, ioprio)); + if (ret) { + perror("Error while setting I/O priority"); + assert(false); + } + assert(IOPRIO_PRIO_CLASS(ioprio_get(IOPRIO_WHO_PROCESS, tid)) == ioprio_class); + assert(IOPRIO_PRIO_DATA(ioprio_get(IOPRIO_WHO_PROCESS, tid)) == ioprio); + } +#endif + + while (!this_worker->seen_join) { + Task* t; + this_worker->queue->pop(t); + t->seen_join = &this_worker->seen_join; + t->Run(); + delete t; + } + return NULL; + } + + void + TaskScheduler::JoinTask::Run() + { + *seen_join = true; + } + +} // namespace fawn diff --git a/fawnds/task.h b/fawnds/task.h new file mode 100644 index 0000000..2fc2f67 --- /dev/null +++ b/fawnds/task.h @@ -0,0 +1,65 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _TASK_H_ +#define _TASK_H_ + +#include +#include +#include + +namespace fawn { + + // a simple task scheduler with round-robin per-thread queues; we don't use the TBB task scheduler, which is geared towared computation-oriented tasks + + class Task { + public: + virtual ~Task() {} + virtual void Run() = 0; + bool* seen_join; + }; + + class TaskScheduler { + public: + enum cpu_priority_t { + CPU_HIGH, + CPU_NORMAL, + CPU_LOW, + }; + + enum io_priority_t { + IO_REAL_TIME, + IO_BEST_EFFORT, + IO_IDLE, + }; + + TaskScheduler(size_t num_workers, size_t queue_capacity, cpu_priority_t cpu_priority = CPU_NORMAL, io_priority_t io_priority = IO_BEST_EFFORT); + ~TaskScheduler(); + + void enqueue_task(Task* t); + void join(); + + protected: + struct Worker { + TaskScheduler* owner; + pthread_t tid; + tbb::concurrent_bounded_queue* queue; + bool seen_join; + + static void* thread_main(void* arg); + }; + + class JoinTask : public Task { + public: + virtual void Run(); + }; + + private: + cpu_priority_t cpu_priority_; + io_priority_t io_priority_; + std::vector workers_; + //tbb::atomic next_worker_; + tbb::concurrent_bounded_queue shared_queue_; + }; + +} // namespace fawn + +#endif // #ifndef _TASK_H_ diff --git a/fawnds/value.h b/fawnds/value.h new file mode 100644 index 0000000..cceb21a --- /dev/null +++ b/fawnds/value.h @@ -0,0 +1,326 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _VALUE_H_ +#define _VALUE_H_ + +#include "basic_types.h" +#include +#include +#include + +namespace fawn { + + class ConstValue + { + public: + explicit ConstValue() + : data_(NULL), size_(0), capacity_(0), share_count_ptr_(NULL) + { + reset(); + } + + ConstValue(const ConstValue& rhs) + { + copy_from(rhs); + } + + ~ConstValue() + { + // Decendant classes must not define a destructor + reset(); + } + + void reset() + { + if (share_count_ptr_ && !--*share_count_ptr_) { + delete [] data_; + delete share_count_ptr_; + } + data_ = NULL; + size_ = 0; + capacity_ = 0; + share_count_ptr_ = NULL; + } + + ConstValue& operator=(const ConstValue& rhs) + { + reset(); + copy_from(rhs); + return *this; + } + + bool operator==(const ConstValue& rhs) const + { + return size_ == rhs.size_ && memcmp(data_, rhs.data_, size_) == 0; + } + + bool operator!=(const ConstValue& rhs) const + { + return !(*this == rhs); + } + + bool operator<(const ConstValue& rhs) const + { + return compare(rhs) < 0; + } + + bool operator<=(const ConstValue& rhs) const + { + return compare(rhs) <= 0; + } + + bool operator>(const ConstValue& rhs) const + { + return compare(rhs) > 0; + } + + bool operator>=(const ConstValue& rhs) const + { + return compare(rhs) > 0; + } + + int compare(const ConstValue& rhs) const + { + size_t llen = size(); + size_t rlen = rhs.size(); + size_t mlen = llen <= rlen ? llen : rlen; + int cmp = memcmp(data(), rhs.data(), mlen); + if (cmp != 0) + return cmp; + else + return llen - rlen; + } + + const char* data() const { return data_; } + + const size_t& size() const { return size_; } + + std::string str() const { return std::string(data_, size_); } + + const size_t& capacity() const { return capacity_; } + + template + const T& as() const + { + return *reinterpret_cast(data_); + } + + template + bool as_number(T* v) const + { + if (std::tr1::is_unsigned::value) + { + switch (size_) + { + case 1: + *v = static_cast(*reinterpret_cast(data_)); + return true; + case 2: + *v = static_cast(*reinterpret_cast(data_)); + return true; + case 4: + *v = static_cast(*reinterpret_cast(data_)); + return true; + case 8: + *v = static_cast(*reinterpret_cast(data_)); + return true; + } + } + else + { + switch (size_) + { + case 1: + *v = static_cast(*reinterpret_cast(data_)); + return true; + case 2: + *v = static_cast(*reinterpret_cast(data_)); + return true; + case 4: + *v = static_cast(*reinterpret_cast(data_)); + return true; + case 8: + *v = static_cast(*reinterpret_cast(data_)); + return true; + } + } + return false; + } + + template + T as_number(const T& default_v = 0) const + { + T v; + if (as_number(&v)) + return v; + else + return default_v; + } + + protected: + void init_copy(const char* data, size_t size, size_t capacity) + { + // must be called at most once + if (capacity < size) + capacity = size; + data_ = new char[capacity]; + memcpy(const_cast(data_), data, size); + size_ = size; + capacity_ = capacity; + share_count_ptr_ = new size_t(1); + } + + void init_ref(const char* data, size_t size, size_t capacity) + { + // must be called at most once + if (capacity < size) + capacity = size; + data_ = data; + size_ = size; + capacity_ = capacity; + share_count_ptr_ = NULL; + } + + void copy_from(const ConstValue& rhs) + { + // reset() is required if this object was holding data + data_ = rhs.data_; + size_ = rhs.size_; + capacity_ = rhs.capacity_; + if ((share_count_ptr_ = rhs.share_count_ptr_)) + ++*rhs.share_count_ptr_; + } + + const char* data_; + size_t size_; + size_t capacity_; + size_t* share_count_ptr_; + }; + + class Value : public ConstValue + { + public: + explicit Value() : ConstValue() { } + + Value(const Value& rhs) : ConstValue(rhs) { } + + explicit Value(const ConstValue& rhs) : ConstValue(rhs) { } // not safe initialization + + Value& operator=(const Value& rhs) + { + return (*this = static_cast(rhs)); + } + + Value& operator=(const ConstValue& rhs) + { + reset(); + copy_from(rhs); + return *this; + } + + const char* data() const { return data_; } + char* data() { return const_cast(data_); } + + void resize(size_t new_size, bool preserve = true) + { + if (data_ != NULL && (size_ >= new_size || capacity_ >= new_size)) + { + size_ = new_size; + return; + } + + if (preserve) + { + char* new_data = new char[new_size]; + memcpy(new_data, data_, size_); + reset(); + data_ = new_data; + size_ = new_size; + capacity_ = new_size; + share_count_ptr_ = new size_t(1); + } + else + { + reset(); + data_ = new char[new_size]; + size_ = new_size; + capacity_ = new_size; + share_count_ptr_ = new size_t(1); + } + } + + protected: + void init_ref(char* data, size_t size, size_t capacity) + { + ConstValue::init_ref(const_cast(data), size, capacity); + } + }; + + class NewValue : public Value + { + public: + explicit NewValue(const char* data, size_t size, size_t capacity = 0) + { + init_copy(data, size, capacity); + } + + template + explicit NewValue(const T* value) + { + init_copy(reinterpret_cast(value), sizeof(T), sizeof(T)); + } + + explicit NewValue(const std::string& s) + { + init_copy(s.data(), s.size(), s.capacity()); + } + }; + + class RefValue : public Value + { + public: + explicit RefValue(char* data, size_t size, size_t capacity = 0) + { + init_ref(data, size, capacity); + } + + template + explicit RefValue(T* value) + { + init_ref(reinterpret_cast(value), sizeof(T), sizeof(T)); + } + }; + + class ConstRefValue : public ConstValue + { + public: + explicit ConstRefValue(const char* data, size_t size, size_t capacity = 0) + { + init_ref(data, size, capacity); + } + + template + explicit ConstRefValue(const T* value) + { + init_ref(reinterpret_cast(value), sizeof(T), sizeof(T)); + } + + explicit ConstRefValue(const std::string& s) + { + init_ref(s.data(), s.size(), s.capacity()); + } + }; + + template + class SizedValue : public RefValue + { + public: + explicit SizedValue() : RefValue(buf_, 0, buf_size_) {} + + protected: + static const size_t buf_size_ = Size; + char buf_[buf_size_]; + }; + +} // namespace fawn + +#endif // #ifndef _VALUE_H_ diff --git a/m4/.placeholder b/m4/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/test/Makefile.am b/test/Makefile.am new file mode 100644 index 0000000..9661384 --- /dev/null +++ b/test/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = fawnds diff --git a/test/Makefile.in b/test/Makefile.in new file mode 100644 index 0000000..9e3cf75 --- /dev/null +++ b/test/Makefile.in @@ -0,0 +1,558 @@ +# Makefile.in generated by automake 1.11.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +# 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, +# Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = test +DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/libtool.m4 \ + $(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \ + $(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \ + $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +SOURCES = +DIST_SOURCES = +RECURSIVE_TARGETS = all-recursive check-recursive dvi-recursive \ + html-recursive info-recursive install-data-recursive \ + install-dvi-recursive install-exec-recursive \ + install-html-recursive install-info-recursive \ + install-pdf-recursive install-ps-recursive install-recursive \ + installcheck-recursive installdirs-recursive pdf-recursive \ + ps-recursive uninstall-recursive +RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \ + distclean-recursive maintainer-clean-recursive +AM_RECURSIVE_TARGETS = $(RECURSIVE_TARGETS:-recursive=) \ + $(RECURSIVE_CLEAN_TARGETS:-recursive=) tags TAGS ctags CTAGS \ + distdir +ETAGS = etags +CTAGS = ctags +DIST_SUBDIRS = $(SUBDIRS) +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +am__relativize = \ + dir0=`pwd`; \ + sed_first='s,^\([^/]*\)/.*$$,\1,'; \ + sed_rest='s,^[^/]*/*,,'; \ + sed_last='s,^.*/\([^/]*\)$$,\1,'; \ + sed_butlast='s,/*[^/]*$$,,'; \ + while test -n "$$dir1"; do \ + first=`echo "$$dir1" | sed -e "$$sed_first"`; \ + if test "$$first" != "."; then \ + if test "$$first" = ".."; then \ + dir2=`echo "$$dir0" | sed -e "$$sed_last"`/"$$dir2"; \ + dir0=`echo "$$dir0" | sed -e "$$sed_butlast"`; \ + else \ + first2=`echo "$$dir2" | sed -e "$$sed_first"`; \ + if test "$$first2" = "$$first"; then \ + dir2=`echo "$$dir2" | sed -e "$$sed_rest"`; \ + else \ + dir2="../$$dir2"; \ + fi; \ + dir0="$$dir0"/"$$first"; \ + fi; \ + fi; \ + dir1=`echo "$$dir1" | sed -e "$$sed_rest"`; \ + done; \ + reldir="$$dir2" +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +MAKEINFO = @MAKEINFO@ +MKDIR_P = @MKDIR_P@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +RANLIB = @RANLIB@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +VERSION = @VERSION@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +lt_ECHO = @lt_ECHO@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +sysconfdir = @sysconfdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +SUBDIRS = fawnds +all: all-recursive + +.SUFFIXES: +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu test/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --gnu test/Makefile +.PRECIOUS: Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +# This directory's subdirectories are mostly independent; you can cd +# into them and run `make' without going through this Makefile. +# To change the values of `make' variables: instead of editing Makefiles, +# (1) if the variable is set in `config.status', edit `config.status' +# (which will cause the Makefiles to be regenerated when you run `make'); +# (2) otherwise, pass the desired values on the `make' command line. +$(RECURSIVE_TARGETS): + @fail= failcom='exit 1'; \ + for f in x $$MAKEFLAGS; do \ + case $$f in \ + *=* | --[!k]*);; \ + *k*) failcom='fail=yes';; \ + esac; \ + done; \ + dot_seen=no; \ + target=`echo $@ | sed s/-recursive//`; \ + list='$(SUBDIRS)'; for subdir in $$list; do \ + echo "Making $$target in $$subdir"; \ + if test "$$subdir" = "."; then \ + dot_seen=yes; \ + local_target="$$target-am"; \ + else \ + local_target="$$target"; \ + fi; \ + ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \ + || eval $$failcom; \ + done; \ + if test "$$dot_seen" = "no"; then \ + $(MAKE) $(AM_MAKEFLAGS) "$$target-am" || exit 1; \ + fi; test -z "$$fail" + +$(RECURSIVE_CLEAN_TARGETS): + @fail= failcom='exit 1'; \ + for f in x $$MAKEFLAGS; do \ + case $$f in \ + *=* | --[!k]*);; \ + *k*) failcom='fail=yes';; \ + esac; \ + done; \ + dot_seen=no; \ + case "$@" in \ + distclean-* | maintainer-clean-*) list='$(DIST_SUBDIRS)' ;; \ + *) list='$(SUBDIRS)' ;; \ + esac; \ + rev=''; for subdir in $$list; do \ + if test "$$subdir" = "."; then :; else \ + rev="$$subdir $$rev"; \ + fi; \ + done; \ + rev="$$rev ."; \ + target=`echo $@ | sed s/-recursive//`; \ + for subdir in $$rev; do \ + echo "Making $$target in $$subdir"; \ + if test "$$subdir" = "."; then \ + local_target="$$target-am"; \ + else \ + local_target="$$target"; \ + fi; \ + ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \ + || eval $$failcom; \ + done && test -z "$$fail" +tags-recursive: + list='$(SUBDIRS)'; for subdir in $$list; do \ + test "$$subdir" = . || ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) tags); \ + done +ctags-recursive: + list='$(SUBDIRS)'; for subdir in $$list; do \ + test "$$subdir" = . || ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) ctags); \ + done + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + mkid -fID $$unique +tags: TAGS + +TAGS: tags-recursive $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + set x; \ + here=`pwd`; \ + if ($(ETAGS) --etags-include --version) >/dev/null 2>&1; then \ + include_option=--etags-include; \ + empty_fix=.; \ + else \ + include_option=--include; \ + empty_fix=; \ + fi; \ + list='$(SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + test ! -f $$subdir/TAGS || \ + set "$$@" "$$include_option=$$here/$$subdir/TAGS"; \ + fi; \ + done; \ + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: CTAGS +CTAGS: ctags-recursive $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done + @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + test -d "$(distdir)/$$subdir" \ + || $(MKDIR_P) "$(distdir)/$$subdir" \ + || exit 1; \ + fi; \ + done + @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + dir1=$$subdir; dir2="$(distdir)/$$subdir"; \ + $(am__relativize); \ + new_distdir=$$reldir; \ + dir1=$$subdir; dir2="$(top_distdir)"; \ + $(am__relativize); \ + new_top_distdir=$$reldir; \ + echo " (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) top_distdir="$$new_top_distdir" distdir="$$new_distdir" \\"; \ + echo " am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir)"; \ + ($(am__cd) $$subdir && \ + $(MAKE) $(AM_MAKEFLAGS) \ + top_distdir="$$new_top_distdir" \ + distdir="$$new_distdir" \ + am__remove_distdir=: \ + am__skip_length_check=: \ + am__skip_mode_fix=: \ + distdir) \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-recursive +all-am: Makefile +installdirs: installdirs-recursive +installdirs-am: +install: install-recursive +install-exec: install-exec-recursive +install-data: install-data-recursive +uninstall: uninstall-recursive + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-recursive +install-strip: + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + `test -z '$(STRIP)' || \ + echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-recursive + +clean-am: clean-generic clean-libtool mostlyclean-am + +distclean: distclean-recursive + -rm -f Makefile +distclean-am: clean-am distclean-generic distclean-tags + +dvi: dvi-recursive + +dvi-am: + +html: html-recursive + +html-am: + +info: info-recursive + +info-am: + +install-data-am: + +install-dvi: install-dvi-recursive + +install-dvi-am: + +install-exec-am: + +install-html: install-html-recursive + +install-html-am: + +install-info: install-info-recursive + +install-info-am: + +install-man: + +install-pdf: install-pdf-recursive + +install-pdf-am: + +install-ps: install-ps-recursive + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-recursive + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-recursive + +mostlyclean-am: mostlyclean-generic mostlyclean-libtool + +pdf: pdf-recursive + +pdf-am: + +ps: ps-recursive + +ps-am: + +uninstall-am: + +.MAKE: $(RECURSIVE_CLEAN_TARGETS) $(RECURSIVE_TARGETS) ctags-recursive \ + install-am install-strip tags-recursive + +.PHONY: $(RECURSIVE_CLEAN_TARGETS) $(RECURSIVE_TARGETS) CTAGS GTAGS \ + all all-am check check-am clean clean-generic clean-libtool \ + ctags ctags-recursive distclean distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-data \ + install-data-am install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-man install-pdf install-pdf-am \ + install-ps install-ps-am install-strip installcheck \ + installcheck-am installdirs installdirs-am maintainer-clean \ + maintainer-clean-generic mostlyclean mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-recursive \ + uninstall uninstall-am + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/test/fawnds/Makefile.am b/test/fawnds/Makefile.am new file mode 100644 index 0000000..b75b9a1 --- /dev/null +++ b/test/fawnds/Makefile.am @@ -0,0 +1,118 @@ +#hashdb_test code +noinst_PROGRAMS = testFawnDS testIterator testTrie testCuckoo testCombi testByYCSBWorkload benchCuckoo benchSemiRandomWrite benchStores preprocessTrace +testFawnDS_SOURCES = testFawnDS.cc +testFawnDS_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +testFawnDS_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +testIterator_SOURCES = testIterator.cc +testIterator_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +testIterator_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +testTrie_SOURCES = testTrie.cc +testTrie_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +testTrie_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +testCuckoo_SOURCES = testCuckoo.cc +testCuckoo_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +testCuckoo_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +testCombi_SOURCES = testCombi.cc +testCombi_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +testCombi_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + + +testByYCSBWorkload_SOURCES= testByYCSBWorkload.cc +testByYCSBWorkload_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +testByYCSBWorkload_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +benchCuckoo_SOURCES = benchCuckoo.cc +benchCuckoo_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +benchCuckoo_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +benchSemiRandomWrite_SOURCES = benchSemiRandomWrite.cc +benchSemiRandomWrite_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +benchSemiRandomWrite_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +benchStores_SOURCES = benchStores.cc +benchStores_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +benchStores_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + + +preprocessTrace_SOURCES= preprocessTrace.cc + +preprocessTrace_CPPFLAGS = + +preprocessTrace_LDADD = + diff --git a/test/fawnds/Makefile.in b/test/fawnds/Makefile.in new file mode 100644 index 0000000..a2ac15c --- /dev/null +++ b/test/fawnds/Makefile.in @@ -0,0 +1,810 @@ +# Makefile.in generated by automake 1.11.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +# 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, +# Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +noinst_PROGRAMS = testFawnDS$(EXEEXT) testIterator$(EXEEXT) \ + testTrie$(EXEEXT) testCuckoo$(EXEEXT) testCombi$(EXEEXT) \ + testByYCSBWorkload$(EXEEXT) benchCuckoo$(EXEEXT) \ + benchSemiRandomWrite$(EXEEXT) benchStores$(EXEEXT) \ + preprocessTrace$(EXEEXT) +subdir = test/fawnds +DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/libtool.m4 \ + $(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \ + $(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \ + $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +PROGRAMS = $(noinst_PROGRAMS) +am_benchCuckoo_OBJECTS = benchCuckoo-benchCuckoo.$(OBJEXT) +benchCuckoo_OBJECTS = $(am_benchCuckoo_OBJECTS) +benchCuckoo_DEPENDENCIES = $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la +am_benchSemiRandomWrite_OBJECTS = \ + benchSemiRandomWrite-benchSemiRandomWrite.$(OBJEXT) +benchSemiRandomWrite_OBJECTS = $(am_benchSemiRandomWrite_OBJECTS) +benchSemiRandomWrite_DEPENDENCIES = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la +am_benchStores_OBJECTS = benchStores-benchStores.$(OBJEXT) +benchStores_OBJECTS = $(am_benchStores_OBJECTS) +benchStores_DEPENDENCIES = $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la +am_preprocessTrace_OBJECTS = \ + preprocessTrace-preprocessTrace.$(OBJEXT) +preprocessTrace_OBJECTS = $(am_preprocessTrace_OBJECTS) +preprocessTrace_DEPENDENCIES = +am_testByYCSBWorkload_OBJECTS = \ + testByYCSBWorkload-testByYCSBWorkload.$(OBJEXT) +testByYCSBWorkload_OBJECTS = $(am_testByYCSBWorkload_OBJECTS) +testByYCSBWorkload_DEPENDENCIES = $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la +am_testCombi_OBJECTS = testCombi-testCombi.$(OBJEXT) +testCombi_OBJECTS = $(am_testCombi_OBJECTS) +testCombi_DEPENDENCIES = $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la +am_testCuckoo_OBJECTS = testCuckoo-testCuckoo.$(OBJEXT) +testCuckoo_OBJECTS = $(am_testCuckoo_OBJECTS) +testCuckoo_DEPENDENCIES = $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la +am_testFawnDS_OBJECTS = testFawnDS-testFawnDS.$(OBJEXT) +testFawnDS_OBJECTS = $(am_testFawnDS_OBJECTS) +testFawnDS_DEPENDENCIES = $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la +am_testIterator_OBJECTS = testIterator-testIterator.$(OBJEXT) +testIterator_OBJECTS = $(am_testIterator_OBJECTS) +testIterator_DEPENDENCIES = $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la +am_testTrie_OBJECTS = testTrie-testTrie.$(OBJEXT) +testTrie_OBJECTS = $(am_testTrie_OBJECTS) +testTrie_DEPENDENCIES = $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__depfiles_maybe = depfiles +am__mv = mv -f +CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +LTCXXCOMPILE = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +CXXLD = $(CXX) +CXXLINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=link $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +SOURCES = $(benchCuckoo_SOURCES) $(benchSemiRandomWrite_SOURCES) \ + $(benchStores_SOURCES) $(preprocessTrace_SOURCES) \ + $(testByYCSBWorkload_SOURCES) $(testCombi_SOURCES) \ + $(testCuckoo_SOURCES) $(testFawnDS_SOURCES) \ + $(testIterator_SOURCES) $(testTrie_SOURCES) +DIST_SOURCES = $(benchCuckoo_SOURCES) $(benchSemiRandomWrite_SOURCES) \ + $(benchStores_SOURCES) $(preprocessTrace_SOURCES) \ + $(testByYCSBWorkload_SOURCES) $(testCombi_SOURCES) \ + $(testCuckoo_SOURCES) $(testFawnDS_SOURCES) \ + $(testIterator_SOURCES) $(testTrie_SOURCES) +ETAGS = etags +CTAGS = ctags +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +MAKEINFO = @MAKEINFO@ +MKDIR_P = @MKDIR_P@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +RANLIB = @RANLIB@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +VERSION = @VERSION@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +lt_ECHO = @lt_ECHO@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +sysconfdir = @sysconfdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +testFawnDS_SOURCES = testFawnDS.cc +testFawnDS_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +testFawnDS_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +testIterator_SOURCES = testIterator.cc +testIterator_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +testIterator_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +testTrie_SOURCES = testTrie.cc +testTrie_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +testTrie_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +testCuckoo_SOURCES = testCuckoo.cc +testCuckoo_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +testCuckoo_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +testCombi_SOURCES = testCombi.cc +testCombi_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +testCombi_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +testByYCSBWorkload_SOURCES = testByYCSBWorkload.cc +testByYCSBWorkload_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +testByYCSBWorkload_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +benchCuckoo_SOURCES = benchCuckoo.cc +benchCuckoo_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +benchCuckoo_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +benchSemiRandomWrite_SOURCES = benchSemiRandomWrite.cc +benchSemiRandomWrite_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +benchSemiRandomWrite_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +benchStores_SOURCES = benchStores.cc +benchStores_CPPFLAGS = \ + -I$(top_srcdir)/utils \ + -I$(top_srcdir)/fawnds \ + -I$(top_builddir)/fawnds \ + -I$(top_builddir)/fawnds/gen-cpp + +benchStores_LDADD = \ + $(top_builddir)/fawnds/libfawnds.la \ + $(top_builddir)/utils/libfawnkvutils.la \ + $(THRIFT_LIBS) + +preprocessTrace_SOURCES = preprocessTrace.cc +preprocessTrace_CPPFLAGS = +preprocessTrace_LDADD = +all: all-am + +.SUFFIXES: +.SUFFIXES: .cc .lo .o .obj +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu test/fawnds/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --gnu test/fawnds/Makefile +.PRECIOUS: Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list +benchCuckoo$(EXEEXT): $(benchCuckoo_OBJECTS) $(benchCuckoo_DEPENDENCIES) + @rm -f benchCuckoo$(EXEEXT) + $(CXXLINK) $(benchCuckoo_OBJECTS) $(benchCuckoo_LDADD) $(LIBS) +benchSemiRandomWrite$(EXEEXT): $(benchSemiRandomWrite_OBJECTS) $(benchSemiRandomWrite_DEPENDENCIES) + @rm -f benchSemiRandomWrite$(EXEEXT) + $(CXXLINK) $(benchSemiRandomWrite_OBJECTS) $(benchSemiRandomWrite_LDADD) $(LIBS) +benchStores$(EXEEXT): $(benchStores_OBJECTS) $(benchStores_DEPENDENCIES) + @rm -f benchStores$(EXEEXT) + $(CXXLINK) $(benchStores_OBJECTS) $(benchStores_LDADD) $(LIBS) +preprocessTrace$(EXEEXT): $(preprocessTrace_OBJECTS) $(preprocessTrace_DEPENDENCIES) + @rm -f preprocessTrace$(EXEEXT) + $(CXXLINK) $(preprocessTrace_OBJECTS) $(preprocessTrace_LDADD) $(LIBS) +testByYCSBWorkload$(EXEEXT): $(testByYCSBWorkload_OBJECTS) $(testByYCSBWorkload_DEPENDENCIES) + @rm -f testByYCSBWorkload$(EXEEXT) + $(CXXLINK) $(testByYCSBWorkload_OBJECTS) $(testByYCSBWorkload_LDADD) $(LIBS) +testCombi$(EXEEXT): $(testCombi_OBJECTS) $(testCombi_DEPENDENCIES) + @rm -f testCombi$(EXEEXT) + $(CXXLINK) $(testCombi_OBJECTS) $(testCombi_LDADD) $(LIBS) +testCuckoo$(EXEEXT): $(testCuckoo_OBJECTS) $(testCuckoo_DEPENDENCIES) + @rm -f testCuckoo$(EXEEXT) + $(CXXLINK) $(testCuckoo_OBJECTS) $(testCuckoo_LDADD) $(LIBS) +testFawnDS$(EXEEXT): $(testFawnDS_OBJECTS) $(testFawnDS_DEPENDENCIES) + @rm -f testFawnDS$(EXEEXT) + $(CXXLINK) $(testFawnDS_OBJECTS) $(testFawnDS_LDADD) $(LIBS) +testIterator$(EXEEXT): $(testIterator_OBJECTS) $(testIterator_DEPENDENCIES) + @rm -f testIterator$(EXEEXT) + $(CXXLINK) $(testIterator_OBJECTS) $(testIterator_LDADD) $(LIBS) +testTrie$(EXEEXT): $(testTrie_OBJECTS) $(testTrie_DEPENDENCIES) + @rm -f testTrie$(EXEEXT) + $(CXXLINK) $(testTrie_OBJECTS) $(testTrie_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/benchCuckoo-benchCuckoo.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/benchSemiRandomWrite-benchSemiRandomWrite.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/benchStores-benchStores.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/preprocessTrace-preprocessTrace.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/testByYCSBWorkload-testByYCSBWorkload.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/testCombi-testCombi.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/testCuckoo-testCuckoo.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/testFawnDS-testFawnDS.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/testIterator-testIterator.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/testTrie-testTrie.Po@am__quote@ + +.cc.o: +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ $< + +.cc.obj: +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.cc.lo: +@am__fastdepCXX_TRUE@ $(LTCXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $< + +benchCuckoo-benchCuckoo.o: benchCuckoo.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(benchCuckoo_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT benchCuckoo-benchCuckoo.o -MD -MP -MF $(DEPDIR)/benchCuckoo-benchCuckoo.Tpo -c -o benchCuckoo-benchCuckoo.o `test -f 'benchCuckoo.cc' || echo '$(srcdir)/'`benchCuckoo.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/benchCuckoo-benchCuckoo.Tpo $(DEPDIR)/benchCuckoo-benchCuckoo.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='benchCuckoo.cc' object='benchCuckoo-benchCuckoo.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(benchCuckoo_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o benchCuckoo-benchCuckoo.o `test -f 'benchCuckoo.cc' || echo '$(srcdir)/'`benchCuckoo.cc + +benchCuckoo-benchCuckoo.obj: benchCuckoo.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(benchCuckoo_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT benchCuckoo-benchCuckoo.obj -MD -MP -MF $(DEPDIR)/benchCuckoo-benchCuckoo.Tpo -c -o benchCuckoo-benchCuckoo.obj `if test -f 'benchCuckoo.cc'; then $(CYGPATH_W) 'benchCuckoo.cc'; else $(CYGPATH_W) '$(srcdir)/benchCuckoo.cc'; fi` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/benchCuckoo-benchCuckoo.Tpo $(DEPDIR)/benchCuckoo-benchCuckoo.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='benchCuckoo.cc' object='benchCuckoo-benchCuckoo.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(benchCuckoo_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o benchCuckoo-benchCuckoo.obj `if test -f 'benchCuckoo.cc'; then $(CYGPATH_W) 'benchCuckoo.cc'; else $(CYGPATH_W) '$(srcdir)/benchCuckoo.cc'; fi` + +benchSemiRandomWrite-benchSemiRandomWrite.o: benchSemiRandomWrite.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(benchSemiRandomWrite_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT benchSemiRandomWrite-benchSemiRandomWrite.o -MD -MP -MF $(DEPDIR)/benchSemiRandomWrite-benchSemiRandomWrite.Tpo -c -o benchSemiRandomWrite-benchSemiRandomWrite.o `test -f 'benchSemiRandomWrite.cc' || echo '$(srcdir)/'`benchSemiRandomWrite.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/benchSemiRandomWrite-benchSemiRandomWrite.Tpo $(DEPDIR)/benchSemiRandomWrite-benchSemiRandomWrite.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='benchSemiRandomWrite.cc' object='benchSemiRandomWrite-benchSemiRandomWrite.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(benchSemiRandomWrite_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o benchSemiRandomWrite-benchSemiRandomWrite.o `test -f 'benchSemiRandomWrite.cc' || echo '$(srcdir)/'`benchSemiRandomWrite.cc + +benchSemiRandomWrite-benchSemiRandomWrite.obj: benchSemiRandomWrite.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(benchSemiRandomWrite_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT benchSemiRandomWrite-benchSemiRandomWrite.obj -MD -MP -MF $(DEPDIR)/benchSemiRandomWrite-benchSemiRandomWrite.Tpo -c -o benchSemiRandomWrite-benchSemiRandomWrite.obj `if test -f 'benchSemiRandomWrite.cc'; then $(CYGPATH_W) 'benchSemiRandomWrite.cc'; else $(CYGPATH_W) '$(srcdir)/benchSemiRandomWrite.cc'; fi` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/benchSemiRandomWrite-benchSemiRandomWrite.Tpo $(DEPDIR)/benchSemiRandomWrite-benchSemiRandomWrite.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='benchSemiRandomWrite.cc' object='benchSemiRandomWrite-benchSemiRandomWrite.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(benchSemiRandomWrite_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o benchSemiRandomWrite-benchSemiRandomWrite.obj `if test -f 'benchSemiRandomWrite.cc'; then $(CYGPATH_W) 'benchSemiRandomWrite.cc'; else $(CYGPATH_W) '$(srcdir)/benchSemiRandomWrite.cc'; fi` + +benchStores-benchStores.o: benchStores.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(benchStores_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT benchStores-benchStores.o -MD -MP -MF $(DEPDIR)/benchStores-benchStores.Tpo -c -o benchStores-benchStores.o `test -f 'benchStores.cc' || echo '$(srcdir)/'`benchStores.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/benchStores-benchStores.Tpo $(DEPDIR)/benchStores-benchStores.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='benchStores.cc' object='benchStores-benchStores.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(benchStores_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o benchStores-benchStores.o `test -f 'benchStores.cc' || echo '$(srcdir)/'`benchStores.cc + +benchStores-benchStores.obj: benchStores.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(benchStores_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT benchStores-benchStores.obj -MD -MP -MF $(DEPDIR)/benchStores-benchStores.Tpo -c -o benchStores-benchStores.obj `if test -f 'benchStores.cc'; then $(CYGPATH_W) 'benchStores.cc'; else $(CYGPATH_W) '$(srcdir)/benchStores.cc'; fi` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/benchStores-benchStores.Tpo $(DEPDIR)/benchStores-benchStores.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='benchStores.cc' object='benchStores-benchStores.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(benchStores_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o benchStores-benchStores.obj `if test -f 'benchStores.cc'; then $(CYGPATH_W) 'benchStores.cc'; else $(CYGPATH_W) '$(srcdir)/benchStores.cc'; fi` + +preprocessTrace-preprocessTrace.o: preprocessTrace.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(preprocessTrace_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT preprocessTrace-preprocessTrace.o -MD -MP -MF $(DEPDIR)/preprocessTrace-preprocessTrace.Tpo -c -o preprocessTrace-preprocessTrace.o `test -f 'preprocessTrace.cc' || echo '$(srcdir)/'`preprocessTrace.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/preprocessTrace-preprocessTrace.Tpo $(DEPDIR)/preprocessTrace-preprocessTrace.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='preprocessTrace.cc' object='preprocessTrace-preprocessTrace.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(preprocessTrace_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o preprocessTrace-preprocessTrace.o `test -f 'preprocessTrace.cc' || echo '$(srcdir)/'`preprocessTrace.cc + +preprocessTrace-preprocessTrace.obj: preprocessTrace.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(preprocessTrace_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT preprocessTrace-preprocessTrace.obj -MD -MP -MF $(DEPDIR)/preprocessTrace-preprocessTrace.Tpo -c -o preprocessTrace-preprocessTrace.obj `if test -f 'preprocessTrace.cc'; then $(CYGPATH_W) 'preprocessTrace.cc'; else $(CYGPATH_W) '$(srcdir)/preprocessTrace.cc'; fi` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/preprocessTrace-preprocessTrace.Tpo $(DEPDIR)/preprocessTrace-preprocessTrace.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='preprocessTrace.cc' object='preprocessTrace-preprocessTrace.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(preprocessTrace_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o preprocessTrace-preprocessTrace.obj `if test -f 'preprocessTrace.cc'; then $(CYGPATH_W) 'preprocessTrace.cc'; else $(CYGPATH_W) '$(srcdir)/preprocessTrace.cc'; fi` + +testByYCSBWorkload-testByYCSBWorkload.o: testByYCSBWorkload.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testByYCSBWorkload_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT testByYCSBWorkload-testByYCSBWorkload.o -MD -MP -MF $(DEPDIR)/testByYCSBWorkload-testByYCSBWorkload.Tpo -c -o testByYCSBWorkload-testByYCSBWorkload.o `test -f 'testByYCSBWorkload.cc' || echo '$(srcdir)/'`testByYCSBWorkload.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/testByYCSBWorkload-testByYCSBWorkload.Tpo $(DEPDIR)/testByYCSBWorkload-testByYCSBWorkload.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='testByYCSBWorkload.cc' object='testByYCSBWorkload-testByYCSBWorkload.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testByYCSBWorkload_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o testByYCSBWorkload-testByYCSBWorkload.o `test -f 'testByYCSBWorkload.cc' || echo '$(srcdir)/'`testByYCSBWorkload.cc + +testByYCSBWorkload-testByYCSBWorkload.obj: testByYCSBWorkload.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testByYCSBWorkload_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT testByYCSBWorkload-testByYCSBWorkload.obj -MD -MP -MF $(DEPDIR)/testByYCSBWorkload-testByYCSBWorkload.Tpo -c -o testByYCSBWorkload-testByYCSBWorkload.obj `if test -f 'testByYCSBWorkload.cc'; then $(CYGPATH_W) 'testByYCSBWorkload.cc'; else $(CYGPATH_W) '$(srcdir)/testByYCSBWorkload.cc'; fi` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/testByYCSBWorkload-testByYCSBWorkload.Tpo $(DEPDIR)/testByYCSBWorkload-testByYCSBWorkload.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='testByYCSBWorkload.cc' object='testByYCSBWorkload-testByYCSBWorkload.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testByYCSBWorkload_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o testByYCSBWorkload-testByYCSBWorkload.obj `if test -f 'testByYCSBWorkload.cc'; then $(CYGPATH_W) 'testByYCSBWorkload.cc'; else $(CYGPATH_W) '$(srcdir)/testByYCSBWorkload.cc'; fi` + +testCombi-testCombi.o: testCombi.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testCombi_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT testCombi-testCombi.o -MD -MP -MF $(DEPDIR)/testCombi-testCombi.Tpo -c -o testCombi-testCombi.o `test -f 'testCombi.cc' || echo '$(srcdir)/'`testCombi.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/testCombi-testCombi.Tpo $(DEPDIR)/testCombi-testCombi.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='testCombi.cc' object='testCombi-testCombi.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testCombi_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o testCombi-testCombi.o `test -f 'testCombi.cc' || echo '$(srcdir)/'`testCombi.cc + +testCombi-testCombi.obj: testCombi.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testCombi_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT testCombi-testCombi.obj -MD -MP -MF $(DEPDIR)/testCombi-testCombi.Tpo -c -o testCombi-testCombi.obj `if test -f 'testCombi.cc'; then $(CYGPATH_W) 'testCombi.cc'; else $(CYGPATH_W) '$(srcdir)/testCombi.cc'; fi` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/testCombi-testCombi.Tpo $(DEPDIR)/testCombi-testCombi.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='testCombi.cc' object='testCombi-testCombi.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testCombi_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o testCombi-testCombi.obj `if test -f 'testCombi.cc'; then $(CYGPATH_W) 'testCombi.cc'; else $(CYGPATH_W) '$(srcdir)/testCombi.cc'; fi` + +testCuckoo-testCuckoo.o: testCuckoo.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testCuckoo_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT testCuckoo-testCuckoo.o -MD -MP -MF $(DEPDIR)/testCuckoo-testCuckoo.Tpo -c -o testCuckoo-testCuckoo.o `test -f 'testCuckoo.cc' || echo '$(srcdir)/'`testCuckoo.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/testCuckoo-testCuckoo.Tpo $(DEPDIR)/testCuckoo-testCuckoo.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='testCuckoo.cc' object='testCuckoo-testCuckoo.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testCuckoo_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o testCuckoo-testCuckoo.o `test -f 'testCuckoo.cc' || echo '$(srcdir)/'`testCuckoo.cc + +testCuckoo-testCuckoo.obj: testCuckoo.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testCuckoo_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT testCuckoo-testCuckoo.obj -MD -MP -MF $(DEPDIR)/testCuckoo-testCuckoo.Tpo -c -o testCuckoo-testCuckoo.obj `if test -f 'testCuckoo.cc'; then $(CYGPATH_W) 'testCuckoo.cc'; else $(CYGPATH_W) '$(srcdir)/testCuckoo.cc'; fi` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/testCuckoo-testCuckoo.Tpo $(DEPDIR)/testCuckoo-testCuckoo.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='testCuckoo.cc' object='testCuckoo-testCuckoo.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testCuckoo_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o testCuckoo-testCuckoo.obj `if test -f 'testCuckoo.cc'; then $(CYGPATH_W) 'testCuckoo.cc'; else $(CYGPATH_W) '$(srcdir)/testCuckoo.cc'; fi` + +testFawnDS-testFawnDS.o: testFawnDS.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testFawnDS_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT testFawnDS-testFawnDS.o -MD -MP -MF $(DEPDIR)/testFawnDS-testFawnDS.Tpo -c -o testFawnDS-testFawnDS.o `test -f 'testFawnDS.cc' || echo '$(srcdir)/'`testFawnDS.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/testFawnDS-testFawnDS.Tpo $(DEPDIR)/testFawnDS-testFawnDS.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='testFawnDS.cc' object='testFawnDS-testFawnDS.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testFawnDS_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o testFawnDS-testFawnDS.o `test -f 'testFawnDS.cc' || echo '$(srcdir)/'`testFawnDS.cc + +testFawnDS-testFawnDS.obj: testFawnDS.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testFawnDS_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT testFawnDS-testFawnDS.obj -MD -MP -MF $(DEPDIR)/testFawnDS-testFawnDS.Tpo -c -o testFawnDS-testFawnDS.obj `if test -f 'testFawnDS.cc'; then $(CYGPATH_W) 'testFawnDS.cc'; else $(CYGPATH_W) '$(srcdir)/testFawnDS.cc'; fi` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/testFawnDS-testFawnDS.Tpo $(DEPDIR)/testFawnDS-testFawnDS.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='testFawnDS.cc' object='testFawnDS-testFawnDS.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testFawnDS_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o testFawnDS-testFawnDS.obj `if test -f 'testFawnDS.cc'; then $(CYGPATH_W) 'testFawnDS.cc'; else $(CYGPATH_W) '$(srcdir)/testFawnDS.cc'; fi` + +testIterator-testIterator.o: testIterator.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testIterator_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT testIterator-testIterator.o -MD -MP -MF $(DEPDIR)/testIterator-testIterator.Tpo -c -o testIterator-testIterator.o `test -f 'testIterator.cc' || echo '$(srcdir)/'`testIterator.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/testIterator-testIterator.Tpo $(DEPDIR)/testIterator-testIterator.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='testIterator.cc' object='testIterator-testIterator.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testIterator_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o testIterator-testIterator.o `test -f 'testIterator.cc' || echo '$(srcdir)/'`testIterator.cc + +testIterator-testIterator.obj: testIterator.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testIterator_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT testIterator-testIterator.obj -MD -MP -MF $(DEPDIR)/testIterator-testIterator.Tpo -c -o testIterator-testIterator.obj `if test -f 'testIterator.cc'; then $(CYGPATH_W) 'testIterator.cc'; else $(CYGPATH_W) '$(srcdir)/testIterator.cc'; fi` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/testIterator-testIterator.Tpo $(DEPDIR)/testIterator-testIterator.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='testIterator.cc' object='testIterator-testIterator.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testIterator_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o testIterator-testIterator.obj `if test -f 'testIterator.cc'; then $(CYGPATH_W) 'testIterator.cc'; else $(CYGPATH_W) '$(srcdir)/testIterator.cc'; fi` + +testTrie-testTrie.o: testTrie.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testTrie_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT testTrie-testTrie.o -MD -MP -MF $(DEPDIR)/testTrie-testTrie.Tpo -c -o testTrie-testTrie.o `test -f 'testTrie.cc' || echo '$(srcdir)/'`testTrie.cc +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/testTrie-testTrie.Tpo $(DEPDIR)/testTrie-testTrie.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='testTrie.cc' object='testTrie-testTrie.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testTrie_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o testTrie-testTrie.o `test -f 'testTrie.cc' || echo '$(srcdir)/'`testTrie.cc + +testTrie-testTrie.obj: testTrie.cc +@am__fastdepCXX_TRUE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testTrie_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT testTrie-testTrie.obj -MD -MP -MF $(DEPDIR)/testTrie-testTrie.Tpo -c -o testTrie-testTrie.obj `if test -f 'testTrie.cc'; then $(CYGPATH_W) 'testTrie.cc'; else $(CYGPATH_W) '$(srcdir)/testTrie.cc'; fi` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/testTrie-testTrie.Tpo $(DEPDIR)/testTrie-testTrie.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='testTrie.cc' object='testTrie-testTrie.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(testTrie_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o testTrie-testTrie.obj `if test -f 'testTrie.cc'; then $(CYGPATH_W) 'testTrie.cc'; else $(CYGPATH_W) '$(srcdir)/testTrie.cc'; fi` + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + mkid -fID $$unique +tags: TAGS + +TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + set x; \ + here=`pwd`; \ + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: CTAGS +CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(PROGRAMS) +installdirs: +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + `test -z '$(STRIP)' || \ + echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstPROGRAMS \ + mostlyclean-am + +distclean: distclean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \ + clean-libtool clean-noinstPROGRAMS ctags distclean \ + distclean-compile distclean-generic distclean-libtool \ + distclean-tags distdir dvi dvi-am html html-am info info-am \ + install install-am install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags uninstall uninstall-am + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/test/fawnds/benchCuckoo.cc b/test/fawnds/benchCuckoo.cc new file mode 100644 index 0000000..5e2e5f6 --- /dev/null +++ b/test/fawnds/benchCuckoo.cc @@ -0,0 +1,202 @@ +#include "fawnds_factory.h" + +#include +#include +#include +#include +#include + +struct kv_pair { + fawn::Value key; + fawn::Value data; +}; +typedef std::vector kv_array_type; + +static void +generate_random_kv(kv_array_type& out_arr, size_t key_len, size_t data_len, size_t size, unsigned int seed = 0) +{ + size_t kv_len = key_len + data_len; + + srand(seed); + char* buf = new char[kv_len * size]; + for (size_t i = 0; i < kv_len * size; i++) + buf[i] = rand() & 0xff; + //buf[i] = (rand() % ('Z' - 'A')) + 'A'; + + out_arr.clear(); + for (size_t i = 0; i < size; i++) { + kv_pair kv; + kv.key = fawn::RefValue(buf + i * kv_len, key_len); + kv.data = fawn::RefValue(buf + i * kv_len + key_len, data_len); + out_arr.push_back(kv); + } +} + +static void +free_kv(kv_array_type& arr) +{ + char* min = reinterpret_cast(-1); + for (size_t i = 0; i < arr.size(); i++) + if (min > arr[i].key.data()) + min = arr[i].key.data(); + delete [] min; + arr.clear(); +} + +static void +swap(kv_array_type& arr, size_t i, size_t j) +{ + kv_pair t = arr[i]; + arr[i] = arr[j]; + arr[j] = t; +} + +static int +cmp(kv_array_type& arr, size_t key_len, size_t i, size_t j) +{ + return memcmp(arr[i].key.data(), arr[j].key.data(), key_len); +} + +static void +quick_sort(kv_array_type& arr, size_t key_len, size_t left, size_t right) +{ + if (left < right) + { + size_t p = left; + size_t i = left; + size_t j = right + 1; + while (i < j) { + while (i < right && cmp(arr, key_len, ++i, p) <= 0); + while (left < j && cmp(arr, key_len, p, --j) <= 0); + if (i < j) + swap(arr, i, j); + } + swap(arr, p, j); + + if (j > 0) + quick_sort(arr, key_len, left, j - 1); + quick_sort(arr, key_len, j + 1, right); + } +} + +static void +sort_keys(kv_array_type& arr, size_t key_len, size_t off, size_t n) +{ + quick_sort(arr, key_len, off, off + n - 1); +} + +/* +static void +print_hex(const char* s, int len) +{ + for (int i = 0; i < len; i++) + printf("%02x", (unsigned char)s[i]); +} +*/ + +using namespace fawn; + +static std::string conf_file = "testConfigs/exp_benchCuckoo.xml"; + +void benchmark(std::string name) +{ + Configuration* config = new Configuration(conf_file); + + size_t key_len = atoi(config->GetStringValue("child::key-len").c_str()); + size_t size = atoi(config->GetStringValue("child::size").c_str()); + assert(key_len != 0); + assert(size != 0); + + config->SetContextNode(name); + + kv_array_type arr; + generate_random_kv(arr, key_len, 0, size); + sort_keys(arr, key_len, 0, size); + + kv_array_type arr2; + generate_random_kv(arr2, key_len, 0, size, 1); + + struct timeval tv; + uint64_t start_time, end_time; + + std::vector indexes; + + size_t index_size = 0; + + { + uint64_t insert_count = 0; + gettimeofday(&tv, NULL); + start_time = tv.tv_sec * 1000000L + tv.tv_usec; + + for (size_t i = 0; i < 1024; i++) { + FawnDS* fawnds = FawnDS_Factory::New(new Configuration(config)); + fawnds->Create(); + + size_t j; + for (j = 0; j < size; j++) { + if (fawnds->Put(arr[j].key, RefValue(&j)) != OK) + break; + } + index_size = j; + insert_count += j; + + indexes.push_back(fawnds); + } + + gettimeofday(&tv, NULL); + end_time = tv.tv_sec * 1000000L + tv.tv_usec; + + printf("%s: %lf inserts/sec\n", name.c_str(), (double)insert_count / (double)(end_time - start_time) * 1000000L); + printf("%s: %zu keys per index\n", name.c_str(), index_size); + } + + for (size_t lookup_mode = 0; lookup_mode < 2; lookup_mode++) + { + uint64_t lookup_count = 0; + gettimeofday(&tv, NULL); + start_time = tv.tv_sec * 1000000L + tv.tv_usec; + + lookup_count = 10000000; + for (size_t k = 0; k < lookup_count; k++) + { + size_t i = rand() % indexes.size(); + size_t j = rand() % index_size; + if (lookup_mode == 0) + { + FawnDS_ConstIterator it = indexes[i]->Find(arr[j].key); + while (!it.IsEnd()) { + if (it->data.as_number() == j) + break; + ++it; + } + } + else + { + FawnDS_ConstIterator it = indexes[i]->Find(arr2[j].key); + while (!it.IsEnd()) { + ++it; + } + } + } + + gettimeofday(&tv, NULL); + end_time = tv.tv_sec * 1000000L + tv.tv_usec; + + printf("%s: %lf lookups/sec (%s)\n", name.c_str(), (double)lookup_count / (double)(end_time - start_time) * 1000000L, lookup_mode == 0 ? "hit" : "miss"); + } + + for (size_t i = 0; i < indexes.size(); i++) + delete indexes[i]; + indexes.clear(); + + free_kv(arr); + free_kv(arr2); + delete config; +} + + +int main(int argc, char** argv) { + benchmark("cuckoo"); + return 0; +} + diff --git a/test/fawnds/benchSemiRandomWrite.cc b/test/fawnds/benchSemiRandomWrite.cc new file mode 100644 index 0000000..a47b9bb --- /dev/null +++ b/test/fawnds/benchSemiRandomWrite.cc @@ -0,0 +1,81 @@ +#include "fawnds_factory.h" +#include +#include +#include +#include +#include +#include +#include + +void benchmark() +{ + fprintf(stdout, "# number_of_files write_speed_MBps\n"); + + for (size_t i = 1; i <= 256; i *= 2) + { + for (size_t j = 0; j < i; j++) { + char buf[1024]; + sprintf(buf, "file%zu", j); + + unlink(buf); + } + sync(); + + fprintf(stdout, "%zu ", i); + fflush(stdout); + + std::vector fds; + for (size_t j = 0; j < i; j++) { + char buf[1024]; + sprintf(buf, "file%zu", j); + + int fd = open(buf, O_WRONLY | O_CREAT | O_NOATIME, 0666); + if (fd == -1) { + perror(""); + return; + } + fds.push_back(fd); + } + + srand(time(NULL)); + + struct timeval tv; + uint64_t start_time, end_time; + + const int64_t total_bytes_to_write = 24L * 1000 * 1000 * 1000; + int64_t remaining_bytes_to_write = total_bytes_to_write; + + gettimeofday(&tv, NULL); + start_time = tv.tv_sec * 1000000L + tv.tv_usec; + + char buf[4096]; + while (remaining_bytes_to_write >= 0) { + uint32_t r = rand(); + for (size_t j = 0; j < sizeof(buf); j += 4) + *reinterpret_cast(buf + j) = r++; + + size_t fd_idx = rand() % i; + if (write(fds[fd_idx], buf, sizeof(buf)) != sizeof(buf)) { + perror(""); + return; + } + + remaining_bytes_to_write -= sizeof(buf); + } + + gettimeofday(&tv, NULL); + end_time = tv.tv_sec * 1000000L + tv.tv_usec; + + fprintf(stdout, "%lf\n", (double)total_bytes_to_write / 1000000. / (double)(end_time - start_time) * 1000000L); + + for (size_t j = 0; j < i; j++) + close(fds[j]); + } +} + + +int main(int argc, char** argv) { + benchmark(); + return 0; +} + diff --git a/test/fawnds/benchStores.cc b/test/fawnds/benchStores.cc new file mode 100644 index 0000000..7fb8ae6 --- /dev/null +++ b/test/fawnds/benchStores.cc @@ -0,0 +1,507 @@ +#include "fawnds_factory.h" +#include "print.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace fawn; + +static std::string conf_file = "testConfigs/exp_benchStores.xml"; + +// TODO: moving these values into configuration file +// assuming 4 partitions (instances) +const int constructor_thread_counts[3] = {4, 4, 4}; +// successful lookup needs some threads (empirically about 16 for good performance) +// while unsuccessful lookup (except SortedStore) makes threshing with too many threads +const int reader_thread_counts[3][2] = {{16, 4}, {16, 4}, {16, 16}}; + + +Configuration* g_main_config; + +std::vector g_log_stores; +std::vector g_hash_stores; +std::vector g_sorted_stores; + +size_t g_key_len; +size_t g_data_len; +size_t g_size; // the size of each of LogStore and HashStore +size_t g_store_count; // the number of LogStores or HashStores +size_t g_merge_size; // how many HashStores will be merged into a SortedStore +size_t g_get_count; // how many operations to use to measure GET speed +size_t g_phase; // the current phase (0 = LogStore, 1 = LogStore->HashStore, 2 = some HashStore->SortedStore) +std::string g_temp_file; // temporary directory for sorter +bool g_successful_lookup; // should the reader make lookups for existing keys or non-exsistent keys +const size_t get_batch_size = 1000; +size_t g_epoch; // the epoch of all the test +const size_t g_max_epoch = 2; + +tbb::atomic g_current_store_i; +tbb::atomic g_current_get_i; +tbb::atomic g_merged_count; +tbb::atomic g_next_id; + +void set_config_param(Configuration* main_config, Configuration* store_config) +{ + char buf[1024]; + + snprintf(buf, sizeof(buf), "%zu", ++g_next_id - 1); + store_config->SetStringValue("child::id", buf); + + store_config->SetStringValue("child::key-len", main_config->GetStringValue("child::key-len")); + store_config->SetStringValue("child::data-len", main_config->GetStringValue("child::data-len")); + if (store_config->ExistsNode("child::size") == 0) { + snprintf(buf, sizeof(buf), "%zu", g_size * g_merge_size * (g_epoch + 1)); + store_config->SetStringValue("child::size", buf); + } +} + +inline uint32_t my_rand_r(uint64_t* v) +{ + // Java's + return (*v = (25214903917ULL * *v + 11ULL) & ((1ULL << 48) - 1)) >> 16; +} + +void generate_kv(char* key, char* data, size_t store_id, size_t key_id) +{ + // determistically generate random key-value pairs + uint64_t seed = store_id * g_size + key_id; + + for (size_t i = 0; i < g_key_len; i += 4) { + *reinterpret_cast(key + i) = my_rand_r(&seed); + } + + if (data) { + uint64_t r = (static_cast(my_rand_r(&seed)) << 32) | my_rand_r(&seed); + uint64_t* p = reinterpret_cast(data); + const uint64_t* p_end = p + (g_data_len / sizeof(uint64_t)); + while (p != p_end) + *p++ = r++; + } +} + +void* constructor_thread_main(void* arg) +{ + assert(g_phase < 3); + + char key[(g_key_len + sizeof(uint64_t) - 1) / sizeof(uint64_t) * sizeof(uint64_t)]; + char data[(g_data_len + sizeof(uint64_t) - 1) / sizeof(uint64_t) * sizeof(uint64_t)]; + + while (true) { + size_t current_store_i = ++g_current_store_i - 1; + if (g_phase == 0 || g_phase == 1) { + if (current_store_i >= g_store_count) + break; + } + else { + if (current_store_i >= (g_store_count + g_merge_size - 1) / g_merge_size) + break; + } + + //printf("creating store %zu out of %zu\n", current_store_i + 1, g_store_count); + + Configuration* store_config = new Configuration(g_main_config, true); + if (g_phase == 0) + store_config->SetContextNode("store0"); + else if (g_phase == 1) + store_config->SetContextNode("store1"); + else + store_config->SetContextNode("store2"); + set_config_param(g_main_config, store_config); + + FawnDS* fawnds = FawnDS_Factory::New(store_config); + assert(fawnds); + + if (fawnds->Create() != OK) + assert(false); + + if (g_phase == 0) { + for (size_t j = 0; j < g_size; j++) { + generate_kv(key, data, current_store_i + g_epoch * g_store_count, j); + + if (fawnds->Put(ConstRefValue(key, g_key_len), ConstRefValue(data, g_data_len)) != OK) + assert(false); + } + g_log_stores[current_store_i] = fawnds; + } + else if (g_phase == 1) { + if (g_log_stores[current_store_i]->ConvertTo(fawnds) != OK) + assert(false); + g_log_stores[current_store_i]->Close(); + g_log_stores[current_store_i]->Destroy(); + delete g_log_stores[current_store_i]; + g_log_stores[current_store_i] = NULL; + g_hash_stores[current_store_i] = fawnds; + } + else { + // TODO: this code is largely borrowed from FawnDS_Combi; + // it would be probably useful to make FawnDS more flexible to expose this sorting functionally to the external. + FawnDS* sorter; + Configuration* sorter_config = new Configuration(); + + char buf[1024]; + + if (sorter_config->CreateNodeAndAppend("type", ".") != 0) + assert(false); + if (sorter_config->SetStringValue("type", "sorter") != 0) + assert(false); + + if (sorter_config->CreateNodeAndAppend("key-len", ".") != 0) + assert(false); + snprintf(buf, sizeof(buf), "%zu", g_key_len); + if (sorter_config->SetStringValue("key-len", buf) != 0) + assert(false); + + if (sorter_config->CreateNodeAndAppend("data-len", ".") != 0) + assert(false); + snprintf(buf, sizeof(buf), "%zu", 1 + g_data_len); + if (sorter_config->SetStringValue("data-len", buf) != 0) + assert(false); + + if (sorter_config->CreateNodeAndAppend("temp-file", ".") != 0) + assert(false); + if (sorter_config->SetStringValue("temp-file", g_temp_file) != 0) + assert(false); + + sorter = FawnDS_Factory::New(sorter_config); + if (!sorter) + assert(false); + if (sorter->Create() != OK) + assert(false); + + for (size_t j = 0; j < g_merge_size; j++) { + if (current_store_i * g_merge_size + j >= g_hash_stores.size()) + break; + FawnDS_ConstIterator it = g_hash_stores[current_store_i * g_merge_size + j]->Enumerate(); + while (!it.IsEnd()) { + Value combined_data; + combined_data.resize(1 + g_data_len); + combined_data.data()[0] = 0; // is delete? + memcpy(combined_data.data() + 1, it->data.data(), it->data.size()); + + if (sorter->Put(it->key, combined_data) != OK) + assert(false); + + ++it; + } + } + + if (sorter->Flush() != OK) + assert(false); + + Value key; + Value data; + bool deleted; + + FawnDS_ConstIterator it_b = g_sorted_stores[current_store_i]->Enumerate(); + FawnDS_ConstIterator it_m = sorter->Enumerate(); + while (true) { + bool select_middle_store; + if (!it_b.IsEnd() && !it_m.IsEnd()) { + if (it_b->key < it_m->key) { + select_middle_store = false; + } + else if (it_b->key > it_m->key) { + select_middle_store = true; + } + else + assert(false); + } + else if (!it_b.IsEnd()) { + select_middle_store = false; + } + else if (!it_m.IsEnd()) { + select_middle_store = true; + } + else + break; + + if (!select_middle_store) { + key = it_b->key; + data = ConstRefValue(it_b->data.data() + 1, g_data_len); + } + else { + key = it_m->key; + data = ConstRefValue(it_m->data.data() + 1, g_data_len); + } + if (fawnds->Put(key, data) != OK) + assert(false); + + g_merged_count++; + if (!select_middle_store) + ++it_b; + else + ++it_m; + } + + delete sorter; + + for (size_t j = 0; j < g_merge_size; j++) { + if (current_store_i * g_merge_size + j >= g_hash_stores.size()) + break; + g_hash_stores[current_store_i * g_merge_size + j]->Close(); + g_hash_stores[current_store_i * g_merge_size + j]->Destroy(); + delete g_hash_stores[current_store_i * g_merge_size + j]; + g_hash_stores[current_store_i * g_merge_size + j] = NULL; + } + + g_sorted_stores[current_store_i]->Close(); + g_sorted_stores[current_store_i]->Destroy(); + delete g_sorted_stores[current_store_i]; + g_sorted_stores[current_store_i] = fawnds; + } + + if (fawnds->Flush() != OK) + assert(false); + } + + return NULL; +} + +void* reader_thread_main(void *arg) +{ + assert(g_phase < 3); + + char key[(g_key_len + sizeof(uint64_t) - 1) / sizeof(uint64_t) * sizeof(uint64_t)]; + Value data; + + static tbb::queuing_mutex mutex; + + while (true) { + size_t current_get_i = (g_current_get_i += get_batch_size) - get_batch_size; + if (current_get_i >= g_get_count) + break; + + uint64_t seed; + { + tbb::queuing_mutex::scoped_lock lock(mutex); + seed = rand(); + } + seed += current_get_i; + + for (size_t trial = 0; trial < get_batch_size; trial++) { + size_t i = my_rand_r(&seed) % g_store_count; + size_t j = my_rand_r(&seed) % g_size; + + if (g_successful_lookup) + generate_kv(key, NULL, i + g_epoch * g_store_count, j); + else + generate_kv(key, NULL, i + g_epoch * g_store_count + g_max_epoch * g_store_count, j); + + FawnDS* fawnds; + + if (g_phase == 0) { + assert(i < g_log_stores.size()); + fawnds = g_log_stores[i]; + } + else if (g_phase == 1) { + assert(i < g_hash_stores.size()); + fawnds = g_hash_stores[i]; + } + else { + assert(i / g_merge_size < g_sorted_stores.size()); + fawnds = g_sorted_stores[i / g_merge_size]; + } + assert(fawnds); + + FawnDS_Return ret = fawnds->Get(ConstRefValue(key, g_key_len), data); + FawnDS_Return expected_ret; + if (g_successful_lookup) + expected_ret = OK; + else + expected_ret = KEY_NOT_FOUND; + + if (ret != expected_ret) { + fprintf(stderr, "unexpected result: %d (expected: %d) #%zu, key:\n", ret, expected_ret, current_get_i); + print_payload(reinterpret_cast(key), g_key_len, 4); + + char expected_data[(g_data_len + sizeof(uint64_t) - 1) / sizeof(uint64_t) * sizeof(uint64_t)]; + if (g_successful_lookup) + generate_kv(key, expected_data, i + g_epoch * g_store_count, j); + else + generate_kv(key, expected_data, i + g_epoch * g_store_count + g_max_epoch * g_store_count, j); + + if (data.size() != g_data_len || memcmp(expected_data, data.data(), g_data_len) != 0) { + fprintf(stderr, " data mismatch: %zu\n", data.size()); + print_payload(reinterpret_cast(data.data()), data.size(), 4); + fprintf(stderr, " expected: %zu\n", g_data_len); + print_payload(reinterpret_cast(expected_data), g_data_len, 4); + } + + assert(false); + } + } + } + + return NULL; +} + +void load_params() +{ + g_main_config = new Configuration(conf_file); + + g_key_len = atoi(g_main_config->GetStringValue("child::key-len").c_str()); + g_data_len = atoi(g_main_config->GetStringValue("child::data-len").c_str()); + g_size = atoi(g_main_config->GetStringValue("child::size").c_str()); + g_store_count = atoi(g_main_config->GetStringValue("child::store-count").c_str()); + g_merge_size = atoi(g_main_config->GetStringValue("child::merge-size").c_str()); + g_get_count = atoi(g_main_config->GetStringValue("child::get-count").c_str()); + g_temp_file = g_main_config->GetStringValue("child::temp-file"); + assert(g_key_len != 0); + assert(g_data_len != 0); + assert(g_size != 0); + assert(g_store_count != 0); + assert(g_merge_size != 0); + assert(g_get_count != 0); + //g_store_count = 64; + //g_get_count = 100000; + + printf("key-len: %zu\n", g_key_len); + printf("data-len: %zu\n", g_data_len); + printf("size: %zu\n", g_size); + printf("store-count: %zu\n", g_store_count); + printf("get-count: %zu\n", g_get_count); + printf("temp-file: %s\n", g_temp_file.c_str()); + printf("\n"); + fflush(stdout); +} + +void benchmark() +{ + g_epoch = 0; + g_next_id = 0; + + g_log_stores.resize(g_store_count, NULL); + g_hash_stores.resize(g_store_count, NULL); + g_sorted_stores.resize((g_store_count + g_merge_size - 1) / g_merge_size, NULL); + for (size_t i = 0; i < g_sorted_stores.size(); i++) { + Configuration* store_config = new Configuration(g_main_config, true); + store_config->SetContextNode("store2"); + set_config_param(g_main_config, store_config); + + FawnDS* fawnds = FawnDS_Factory::New(store_config); + assert(fawnds); + + if (fawnds->Create() != OK) + assert(false); + g_sorted_stores[i] = fawnds; + fawnds->Flush(); + } + + while (g_epoch < g_max_epoch) { + printf("epoch: %zu\n", g_epoch); + fflush(stdout); + + sync(); + + struct timeval tv; + uint64_t start_time, end_time; + + g_phase = 0; + g_merged_count = 0; + + while (g_phase < 3) { + g_current_store_i = 0; + + { + gettimeofday(&tv, NULL); + start_time = tv.tv_sec * 1000000L + tv.tv_usec; + + std::vector tids; + for (size_t k = 0; k < constructor_thread_counts[g_phase]; k++) { + pthread_t tid; + if (pthread_create(&tid, NULL, constructor_thread_main, NULL)) + perror(""); + tids.push_back(tid); + } + + for (size_t k = 0; k < constructor_thread_counts[g_phase]; k++) + pthread_join(tids[k], NULL); + tids.clear(); + + sync(); + + gettimeofday(&tv, NULL); + end_time = tv.tv_sec * 1000000L + tv.tv_usec; + } + + printf("%zu: %lf keys/sec for construction\n", g_phase, (double)(g_store_count * g_size) / (double)(end_time - start_time) * 1000000L); + fflush(stdout); + + //Value status; + //if (g_stores[0]->Status(MEMORY_USE, status) != OK) + // assert(false); + //printf("%zu: MEMORY_USE: %s (one store)\n", g_phase, status.str().c_str()); + //fflush(stdout); + + sleep(10); + + sync(); + + for (int trial = 0; trial < 2; trial++) { + for (int lookup_mode = 0; lookup_mode < 2; lookup_mode++) { + g_current_get_i = 0; + if (lookup_mode == 0) + g_successful_lookup = true; + else + g_successful_lookup = false; + + gettimeofday(&tv, NULL); + start_time = tv.tv_sec * 1000000L + tv.tv_usec; + + std::vector tids; + for (size_t k = 0; k < reader_thread_counts[g_phase][lookup_mode]; k++) { + pthread_t tid; + if (pthread_create(&tid, NULL, reader_thread_main, NULL)) + perror(""); + tids.push_back(tid); + } + + for (size_t k = 0; k < reader_thread_counts[g_phase][lookup_mode]; k++) + pthread_join(tids[k], NULL); + tids.clear(); + + gettimeofday(&tv, NULL); + end_time = tv.tv_sec * 1000000L + tv.tv_usec; + + printf("%zu: %lf GETs/sec (%s)\n", g_phase, (double)g_get_count / (double)(end_time - start_time) * 1000000L, lookup_mode == 0 ? "hit" : "miss"); + fflush(stdout); + } + } + + printf("\n"); + fflush(stdout); + + g_phase++; + } + + printf("\n"); + fflush(stdout); + + assert(g_merged_count == g_store_count * g_size * (g_epoch + 1)); + g_epoch++; + } + + for (size_t i = 0; i < g_sorted_stores.size(); i++) { + g_sorted_stores[i]->Close(); + g_sorted_stores[i]->Destroy(); + delete g_sorted_stores[i]; + g_sorted_stores[i] = NULL; + } + + sync(); + + delete g_main_config; +} + +int main(int argc, char** argv) { + load_params(); + benchmark(); + + return 0; +} + diff --git a/test/fawnds/preprocessTrace.cc b/test/fawnds/preprocessTrace.cc new file mode 100644 index 0000000..38aec48 --- /dev/null +++ b/test/fawnds/preprocessTrace.cc @@ -0,0 +1,79 @@ +#include +#include +#include +#include +#include + +#include "preprocessTrace.h" + +using namespace std; + + + +static void sha1(char hash[20], const char* buf, size_t count) +{ + SHA1(reinterpret_cast(buf), count, reinterpret_cast(hash)); +} + + +int main(int argc, char **argv) { + + if (argc <= 2) { + cout << "usage: preprocess fieldlength output_filename < input_filename" << endl; + exit (1); + } + + //size_t val_len = static_cast(-1); + size_t val_len = atoi(argv[1]); + FILE *fp; + fp = fopen(argv[2], "w"); + + fwrite(&val_len, sizeof(size_t), 1, fp); + + const size_t tmp_size = 1048576; + char* tmp = new char[tmp_size]; + + while (fgets(tmp, tmp_size, stdin)) { + char key[MAXLEN]; + //char val[MAXLEN]; + struct Query q; + + /* + if (val_len == static_cast(-1)) { + if (sscanf(tmp, "\"fieldlength\"=\"%zu\"", &val_len)) { + fwrite(&val_len, sizeof(size_t), 1, fp); + } + } + */ + + if (sscanf(tmp, "INSERT usertable %s [ field", key)){ + q.tp = PUT; + sha1(q.hashed_key, key, strlen(key)); + //const char *pos = index(tmp, '='); + //strncpy(val, pos + 1, val_len); + //val[val_len] = 0; + } else if (sscanf(tmp, "UPDATE usertable %s [ field", key)) { + q.tp = PUT; + sha1(q.hashed_key, key, strlen(key)); + //const char *pos = index(tmp, '='); + //strncpy(val, pos + 1, val_len); + //val[val_len] = 0; + } + else if (sscanf(tmp, "READ usertable %s [", key)) { + q.tp = GET; + sha1(q.hashed_key, key, strlen(key)); + } else + continue; + + if (q.tp == PUT) { + fwrite(&q, sizeof(Query), 1, fp); + //fwrite(val, sizeof(char), val_len, fp); + } else if (q.tp == GET) { + fwrite(&q, sizeof(Query), 1, fp); + } + + + } + delete tmp; + fclose(fp); +} diff --git a/test/fawnds/preprocessTrace.h b/test/fawnds/preprocessTrace.h new file mode 100644 index 0000000..1e84078 --- /dev/null +++ b/test/fawnds/preprocessTrace.h @@ -0,0 +1,16 @@ +#ifndef _PREPROCESSTRACE_H_ +#define _PREPROCESSTRACE_H_ + +#define MAXLEN 1024 +enum QUERY_TYPE{ + PUT=0, + GET, + DELETE, +}; + +struct Query{ + char hashed_key[20]; + char tp; +} __attribute__((__packed__)) ; + +#endif /* _PREPROCESSTRACE_H_ */ diff --git a/test/fawnds/testByYCSBWorkload.cc b/test/fawnds/testByYCSBWorkload.cc new file mode 100644 index 0000000..49ddb65 --- /dev/null +++ b/test/fawnds/testByYCSBWorkload.cc @@ -0,0 +1,314 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include +#include +#include +#include +#include +#include + +#include "fawnds_factory.h" +#include "rate_limiter.h" +#include "global_limits.h" +#include "datastat.h" + +#include "preprocessTrace.h" + +using namespace std; +using namespace tbb; +using namespace fawn; + +#define MAX_QUERIES 1048576 + +int32_t num_threads = 1; +int64_t max_ops_per_sec = 1000000000L; +int64_t convert_rate = 1000000000L; +int64_t merge_rate = 1000000000L; +double successful_get_ratio = 1.; + +FawnDS *h; +size_t val_len; + +RateLimiter *rate_limiter; + +struct Query q_buf[MAX_QUERIES]; +uint64_t num_query_sent = 0; +uint64_t num_query_read = 0; +bool done; + +pthread_mutex_t query_lock; +pthread_mutex_t stat_lock; + +DataStat *latency_put; +DataStat *latency_get; + + +void *query_reader(void *p) { + FILE *fp = (FILE *) p; + size_t n; + + printf("starting thread: query_reader!\n"); + + if (fread(&val_len, sizeof(size_t), 1, fp)) + cout << "val_len=" << val_len << "bytes" << endl; + else + exit(-1); + + while(!feof((FILE*) fp)) { + if (num_query_read < num_query_sent + MAX_QUERIES/2) { + n = fread(&(q_buf[num_query_read % MAX_QUERIES]), sizeof(Query), 1024, fp); + + num_query_read = num_query_read + n; + + //printf("[query_reader] num_query_read=%ld\n", num_query_read); + } else { + //cout << "query_reader is sleeping! num_query_read=" << num_query_read << " num_query_sent=" << num_query_sent << endl; + struct timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = 100 * 1000; + nanosleep(&ts, NULL); + } + } + done = true; + fclose(fp); + printf("killing thread: query_reader!\n"); + pthread_exit(NULL); +} + +void *query_sender(void * id) { + struct Query q; + FawnDS_Return ret; + long t = (long) id; + char val[(MAXLEN + sizeof(uint64_t) - 1) / sizeof(uint64_t) * sizeof(uint64_t)]; + unsigned int data_seed = 0; + struct timespec ts; + double last_time_, current_time_, latency; + printf("starting thread: query_sender%ld!\n", t); + while (1) { + + pthread_mutex_lock(&query_lock); + + if (num_query_sent < num_query_read) { + uint64_t cur = num_query_sent; + num_query_sent ++; + q = q_buf[cur % MAX_QUERIES]; // copy data to ensure no stall access (with some concurrency penalty) + + pthread_mutex_unlock(&query_lock); + + rate_limiter->remove_tokens(1); + if (q.tp == PUT) { + uint64_t r = (static_cast(rand_r(&data_seed)) << 32) | static_cast(rand_r(&data_seed)); + uint64_t* p = reinterpret_cast(val); + const uint64_t* p_end = p + (val_len / sizeof(uint64_t)); + while (p != p_end) + *p++ = r++; + + clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + last_time_ = static_cast(ts.tv_sec) * 1000000000Lu + static_cast(ts.tv_nsec); + ret = h->Put(ConstRefValue(q.hashed_key, 20), + ConstRefValue(val, val_len)); + if (ret != OK) { + printf("error! h->Put() return value=%d, expected=%d, operation%llu\n", + ret, OK, static_cast(cur)); + //exit(1); + } + clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + current_time_ = static_cast(ts.tv_sec) * 1000000000Lu + static_cast(ts.tv_nsec); + latency = current_time_ - last_time_; + + pthread_mutex_lock(&stat_lock); + latency_put->insert(latency); + pthread_mutex_unlock(&stat_lock); + + + + } else if (q.tp == GET) { + FawnDS_Return expected_ret; + if (static_cast(rand()) / static_cast(RAND_MAX) <= successful_get_ratio) + expected_ret = OK; + else { + // make the query to be unsuccessful assuming extremely low key collision + for (size_t i = 0; i < 20; i += 2) + *reinterpret_cast(q.hashed_key + i) = static_cast(rand()); + expected_ret = KEY_NOT_FOUND; + } + + SizedValue read_data; + clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + last_time_ = static_cast(ts.tv_sec) * 1000000000Lu + static_cast(ts.tv_nsec); + ret = h->Get(ConstRefValue(q.hashed_key, 20), read_data); + if (ret != expected_ret) { + printf("error! h->Get() return value=%d, expected=%d, operation%llu\n", + ret, expected_ret, static_cast(cur)); + //exit(1); + } + + clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + current_time_ = static_cast(ts.tv_sec) * 1000000000Lu + static_cast(ts.tv_nsec); + latency = current_time_ - last_time_; + + pthread_mutex_lock(&stat_lock); + latency_get->insert(latency); + pthread_mutex_unlock(&stat_lock); + + } else { + printf("unknown query type %d.\n", q.tp); + } + + } else { + pthread_mutex_unlock(&query_lock); + + if (done) { + break; + } + else { + //cout << "query_sender is sleeping! num_query_read=" << num_query_read << " num_query_sent=" << num_query_sent << endl; + struct timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = 100 * 1000; + nanosleep(&ts, NULL); + } + } + } /* end of while (done) */ + printf("killing thread: query_sender%ld!\n", t); + pthread_exit(NULL); +} /* end of query_sender */ + + +void replay(string recfile) { + pthread_t reader_thread; + pthread_t *sender_threads = new pthread_t[num_threads]; + + // see if the recfile is good + FILE *fp = fopen(recfile.c_str(), "r"); + if (fp == NULL) { + cout << "can not open file " << recfile << endl; + exit(-1); + } + + pthread_mutex_init(&query_lock, 0 ); + pthread_mutex_init(&stat_lock, 0 ); + num_query_read = num_query_sent = 0; + done = false; + + latency_put = new DataStat("PUT-LATENCY(ns)", 0, 100000000, 10000., 10., 1.02); + latency_get = new DataStat("GET-LATENCY(ns)", 0, 100000000, 10000., 10., 1.02); + + // create threads! + pthread_create(&reader_thread, NULL, query_reader, (void *) fp); + struct timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = 1000 * 1000; + nanosleep(&ts, NULL); + for (int t = 0; t < num_threads; t++) { + pthread_create(&sender_threads[t], NULL, query_sender, (void *) t); + } + + // wait for threads + pthread_join(reader_thread, NULL); + for (int t = 0; t < num_threads; t++) { + pthread_join(sender_threads[t], NULL); + } + + // output and destroy everything + printf("total %llu ops in %f sec: tput = %.2f KQPS\n", + static_cast (latency_put->num() + latency_get->num()), + 0.000000001 * (latency_put->sum() + latency_get->sum()), + 1000000. * (latency_put->num() + latency_get->num()) / (latency_put->sum() + latency_get->sum()) ); + + latency_put->summary(); + latency_get->summary(); + + if (latency_put->num()) { + latency_put->cdf(1.02, true); + } + + if (latency_get->num()) { + latency_get->cdf(1.02, true); + } + + pthread_mutex_destroy(&query_lock); + pthread_mutex_destroy(&stat_lock); + delete latency_put; + delete latency_get; + + h->Flush(); + sleep(10); // these sleep() gives time for FawnDS_Monitor to print status messages after the store reaches a steady state + sync(); + sleep(5); +} + + +void usage() { + cout << "usage: testByYCSBWorkload conf_file load_workload_name trans_workload_name [-t num_threads] [-r max_ops_per_sec] [-c convert_rate] [-m merge_rate] [-s successful_get_ratio]" << endl + << "e.g. ./testByYCSBWorkload testConfigs/bdb.xml testWorkloads/update_only" << endl; + +} + +int main(int argc, char **argv) { + + char ch; + while ((ch = getopt(argc, argv, "t:r:c:m:s:")) != -1) + switch (ch) { + case 't': num_threads = atoi(optarg); break; + case 'r': max_ops_per_sec = atoll(optarg); break; + case 'c': convert_rate = atoll(optarg); break; + case 'm': merge_rate = atoll(optarg); break; + case 's': successful_get_ratio = atof(optarg); break; + default: + usage(); + exit(-1); + } + + argc -= optind; + argv += optind; + + if (argc < 3) { + usage(); + exit(-1); + } + + cout << "num_threads: " << num_threads << endl; + cout << "max_ops_per_sec: " << max_ops_per_sec << endl; + cout << "convert_rate: " << convert_rate << endl; + cout << "merge_rate: " << merge_rate << endl; + cout << "successful_get_ratio: " << successful_get_ratio << endl; + + string conf_file = string(argv[0]); + string load_workload = string(argv[1]); + string trans_workload = string(argv[2]); + + // clean dirty pages to reduce anomalies + sync(); + + GlobalLimits::instance().set_convert_rate(convert_rate); + GlobalLimits::instance().set_merge_rate(merge_rate); + + // prepare FAWNDS + h = FawnDS_Factory::New(conf_file); // test_num_records_ + if (h->Create() != OK) { + cout << "cannot create FAWNDS!" << endl; + exit(0); + } + + rate_limiter = new RateLimiter(0, 1000000000L, 1, 1); // virtually unlimted + GlobalLimits::instance().disable(); + + cout << "process load ... " << endl; + replay(load_workload + ".load"); + cout << "process load ... done" << endl; + cout << endl << endl << endl; + + delete rate_limiter; + + rate_limiter = new RateLimiter(0, max_ops_per_sec, 1, 1000000000L / max_ops_per_sec); + GlobalLimits::instance().enable(); + + cout << "process transaction ... " << endl; + replay(trans_workload + ".trans"); + cout << "process transaction ... done" << endl; + + delete rate_limiter; + + h->Close(); + delete h; +} diff --git a/test/fawnds/testCombi.cc b/test/fawnds/testCombi.cc new file mode 100644 index 0000000..6143400 --- /dev/null +++ b/test/fawnds/testCombi.cc @@ -0,0 +1,286 @@ +#include "fawnds_factory.h" +#include "rate_limiter.h" + +#include +#include +#include + +struct kv_pair { + fawn::Value key; + fawn::Value data; +}; +typedef std::vector kv_array_type; + +static void +generate_random_kv(kv_array_type& out_arr, size_t key_len, size_t data_len, size_t size, unsigned int seed = 0) +{ + size_t kv_len = key_len + data_len; + + srand(seed); + char* buf = new char[kv_len * size]; + for (size_t i = 0; i < kv_len * size; i++) + buf[i] = rand() & 0xff; + //buf[i] = (rand() % ('Z' - 'A')) + 'A'; + + out_arr.clear(); + for (size_t i = 0; i < size; i++) { + kv_pair kv; + kv.key = fawn::RefValue(buf + i * kv_len, key_len); + kv.data = fawn::RefValue(buf + i * kv_len + key_len, data_len); + out_arr.push_back(kv); + } +} + +static void +free_kv(kv_array_type& arr) +{ + char* min = reinterpret_cast(-1); + for (size_t i = 0; i < arr.size(); i++) + if (min > arr[i].key.data()) + min = arr[i].key.data(); + delete [] min; + arr.clear(); +} + +/* +static void +print_hex(const char* s, int len) +{ + for (int i = 0; i < len; i++) + printf("%02x", (unsigned char)s[i]); +} +*/ + +namespace fawn +{ + static std::string conf_file = "testConfigs/testCombi.xml"; + + class FawnDS_Combi_Test : public testing::Test + { + protected: + // Code here will be called immediately after the constructor (right before + // each test). + virtual void SetUp() { + Configuration* config = new Configuration(conf_file); + key_len_ = atoi(config->GetStringValue("child::key-len").c_str()); + data_len_ = atoi(config->GetStringValue("child::data-len").c_str()); + size_ = atoi(config->GetStringValue("child::size").c_str()); + assert(key_len_ != 0); + assert(data_len_ != 0); + assert(size_ != 0); + + fawnds_ = FawnDS_Factory::New(config); + fawnds_->Create(); + + ret_data_.resize(0); + } + + // Code in the TearDown() method will be called immediately after each test + // (right before the destructor). + virtual void TearDown() { + delete fawnds_; + } + + // Objects declared here can be used by all tests in the test case for HashDB. + + size_t key_len_; + size_t data_len_; + size_t size_; + + kv_array_type arr_; + + FawnDS* fawnds_; + + Value ret_data_; + }; + + TEST_F(FawnDS_Combi_Test, TestSimpleInsertRetrieve1) { + generate_random_kv(arr_, key_len_, data_len_, 1); + + EXPECT_EQ(OK, fawnds_->Put(arr_[0].key, arr_[0].data)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[0].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[0].data.data(), ret_data_.data(), data_len_)); + + free_kv(arr_); + } + + TEST_F(FawnDS_Combi_Test, TestSimpleInsertRetrieve2) { + generate_random_kv(arr_, key_len_, data_len_, 2); + + arr_[0].key.data()[0] = 5; + arr_[1].key.data()[0] = 7; + + EXPECT_EQ(OK, fawnds_->Put(arr_[0].key, arr_[0].data)); + EXPECT_EQ(OK, fawnds_->Put(arr_[1].key, arr_[1].data)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[0].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[0].data.data(), ret_data_.data(), data_len_)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[1].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[1].data.data(), ret_data_.data(), data_len_)); + + free_kv(arr_); + } + + TEST_F(FawnDS_Combi_Test, TestSimpleInsertRetrieve4) { + generate_random_kv(arr_, key_len_, data_len_, 4); + + arr_[0].key.data()[0] = 5; + arr_[1].key.data()[0] = 7; + arr_[2].key.data()[0] = 3; + arr_[3].key.data()[0] = 9; + + EXPECT_EQ(OK, fawnds_->Put(arr_[0].key, arr_[0].data)); + EXPECT_EQ(OK, fawnds_->Put(arr_[1].key, arr_[1].data)); + EXPECT_EQ(OK, fawnds_->Put(arr_[2].key, arr_[2].data)); + EXPECT_EQ(OK, fawnds_->Put(arr_[3].key, arr_[3].data)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[0].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[0].data.data(), ret_data_.data(), data_len_)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[1].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[1].data.data(), ret_data_.data(), data_len_)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[2].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[2].data.data(), ret_data_.data(), data_len_)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[3].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[3].data.data(), ret_data_.data(), data_len_)); + + free_kv(arr_); + } + + TEST_F(FawnDS_Combi_Test, TestSimpleSortedInsertRetrieveSome) { + generate_random_kv(arr_, key_len_, data_len_, 100); + + for (size_t i = 0; i < 100; i++) + EXPECT_EQ(OK, fawnds_->Put(arr_[i].key, arr_[i].data)); + + for (size_t i = 0; i < 100; i++) + { + EXPECT_EQ(OK, fawnds_->Get(arr_[i].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[i].data.data(), ret_data_.data(), data_len_)); + } + + free_kv(arr_); + } + + TEST_F(FawnDS_Combi_Test, TestSimpleSortedInsertRetrieveSomeSome) { + generate_random_kv(arr_, key_len_, data_len_, 100); + + for (size_t i = 0; i < 100; i++) + EXPECT_EQ(OK, fawnds_->Put(arr_[i].key, arr_[i].data)); + + // update data with the same key + for (size_t i = 0; i < 100; i++) { + arr_[i].data.data()[0] = 255 - arr_[i].data.data()[0]; + EXPECT_EQ(OK, fawnds_->Put(arr_[i].key, arr_[i].data)); + } + + for (size_t i = 0; i < 100; i++) + { + EXPECT_EQ(OK, fawnds_->Get(arr_[i].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[i].data.data(), ret_data_.data(), data_len_)); + } + + free_kv(arr_); + } + + TEST_F(FawnDS_Combi_Test, TestSimpleSortedInsertRetrieveMany) { + generate_random_kv(arr_, key_len_, data_len_, size_); + + int64_t operations_per_sec = 100000L; + RateLimiter rate_limiter(0, operations_per_sec, operations_per_sec / 1000L, 1000000000L / 1000L); + + for (size_t i = 0; i < size_; i++) { + rate_limiter.remove_tokens(1); + EXPECT_EQ(OK, fawnds_->Put(arr_[i].key, arr_[i].data)); + } + + EXPECT_EQ(OK, fawnds_->Flush()); + sleep(5); + + for (size_t i = 0; i < size_; i++) + { + rate_limiter.remove_tokens(1); + EXPECT_EQ(OK, fawnds_->Get(arr_[i].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[i].data.data(), ret_data_.data(), data_len_)); + } + + free_kv(arr_); + } + + TEST_F(FawnDS_Combi_Test, TestSimpleSortedInsertRetrieveManyMany) { + generate_random_kv(arr_, key_len_, data_len_, size_); + + int64_t operations_per_sec = 100000L; + RateLimiter rate_limiter(0, operations_per_sec, operations_per_sec / 1000L, 1000000000L / 1000L); + + for (size_t i = 0; i < size_; i++) { + rate_limiter.remove_tokens(1); + EXPECT_EQ(OK, fawnds_->Put(arr_[i].key, arr_[i].data)); + } + + for (size_t i = 0; i < size_; i++) { + rate_limiter.remove_tokens(1); + arr_[i].data.data()[0] = 255 - arr_[i].data.data()[0]; + EXPECT_EQ(OK, fawnds_->Put(arr_[i].key, arr_[i].data)); + } + + // don't wait so that background merge and Get() happen simultaneously + //EXPECT_EQ(OK, fawnds_->Flush()); + //sleep(5); + + for (size_t i = 0; i < size_; i++) + { + rate_limiter.remove_tokens(1); + EXPECT_EQ(OK, fawnds_->Get(arr_[i].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[i].data.data(), ret_data_.data(), data_len_)); + } + + free_kv(arr_); + } + + TEST_F(FawnDS_Combi_Test, DISABLED_TestSimpleSortedInsertRetrieveManyOnTheFly) { + int64_t operations_per_sec = 100000L; + RateLimiter rate_limiter(0, operations_per_sec, operations_per_sec / 1000L, 1000000000L / 1000L); + + srand(0); + for (size_t i = 0; i < 10000000; i++) { + //rate_limiter.remove_tokens(1); + + char key[key_len_ + data_len_]; + for (size_t i = 0; i < key_len_ + data_len_; i++) + key[i] = rand() & 0xff; + + EXPECT_EQ(OK, fawnds_->Put(RefValue(key, key_len_), RefValue(key + key_len_, data_len_))); + } + + EXPECT_EQ(OK, fawnds_->Flush()); + sleep(5); + } + + TEST_F(FawnDS_Combi_Test, TestFlush) { + EXPECT_EQ(OK, fawnds_->Flush()); + EXPECT_EQ(OK, fawnds_->Flush()); + } + +} // namespace fawn + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/test/fawnds/testConfigs/exp_bdb.xml b/test/fawnds/testConfigs/exp_bdb.xml new file mode 100644 index 0000000..904125c --- /dev/null +++ b/test/fawnds/testConfigs/exp_bdb.xml @@ -0,0 +1,7 @@ + + monitor + + bdb + 0 + ./testFiles/bdb + diff --git a/test/fawnds/testConfigs/exp_benchCuckoo.xml b/test/fawnds/testConfigs/exp_benchCuckoo.xml new file mode 100644 index 0000000..fcf8c9f --- /dev/null +++ b/test/fawnds/testConfigs/exp_benchCuckoo.xml @@ -0,0 +1,11 @@ + + 20 + 131072 + + + cuckoo + 0 + ./testFiles/hashtable + 131072 + + diff --git a/test/fawnds/testConfigs/exp_benchStores.xml b/test/fawnds/testConfigs/exp_benchStores.xml new file mode 100644 index 0000000..1a934d2 --- /dev/null +++ b/test/fawnds/testConfigs/exp_benchStores.xml @@ -0,0 +1,17 @@ + + 20 + 1000 + + 100000 + 1000 + + 32 + + 100000000 + + ./tmp/ + + exp_common.xmlchild::store0. + exp_common.xmlchild::store1. + exp_common.xmlchild::store2. + diff --git a/test/fawnds/testConfigs/exp_common.xml b/test/fawnds/testConfigs/exp_common.xml new file mode 100644 index 0000000..cfd8b22 --- /dev/null +++ b/test/fawnds/testConfigs/exp_common.xml @@ -0,0 +1,67 @@ + + + sf + + + + ./testFiles/front_file_header + + + cuckoo + + ./testFiles/front_hashtable + 131072 + + + + file + + + ./testFiles/front_datastore + + + + + sf + + + + ./testFiles/middle_file_header + + 0 + + + cuckoo + + ./testFiles/middle_hashtable + 131072 + 0 + + + + file + + + ./testFiles/middle_datastore + + + + + sf_ordered_trie + + + + 1 + + 256 + 0 + + + file + + + ./testFiles/back_datastore + + + + diff --git a/test/fawnds/testConfigs/exp_d.xml b/test/fawnds/testConfigs/exp_d.xml new file mode 100644 index 0000000..038ecce --- /dev/null +++ b/test/fawnds/testConfigs/exp_d.xml @@ -0,0 +1,6 @@ + + exp_dis.xmlchild::store. + + child::store/stage-limit0 + + diff --git a/test/fawnds/testConfigs/exp_di.xml b/test/fawnds/testConfigs/exp_di.xml new file mode 100644 index 0000000..e3fd3ac --- /dev/null +++ b/test/fawnds/testConfigs/exp_di.xml @@ -0,0 +1,6 @@ + + exp_dis.xmlchild::store. + + child::store/stage-limit1 + + diff --git a/test/fawnds/testConfigs/exp_dis.xml b/test/fawnds/testConfigs/exp_dis.xml new file mode 100644 index 0000000..071d6fe --- /dev/null +++ b/test/fawnds/testConfigs/exp_dis.xml @@ -0,0 +1,19 @@ + + + combi + + + 20 + 1000 + + ./tmp/ + + 2 + 1 + 0 + + + exp_common.xmlchild::store0child::store + exp_common.xmlchild::store1child::store + exp_common.xmlchild::store2child::store + diff --git a/test/fawnds/testConfigs/exp_ds.xml b/test/fawnds/testConfigs/exp_ds.xml new file mode 100644 index 0000000..3cc1eff --- /dev/null +++ b/test/fawnds/testConfigs/exp_ds.xml @@ -0,0 +1,7 @@ + + exp_dis.xmlchild::store. + + child::store/store1/hashtable/use-offset1 + child::store/store1/keep-datastore-on-conversion1 + + diff --git a/test/fawnds/testConfigs/exp_p16_dis.xml b/test/fawnds/testConfigs/exp_p16_dis.xml new file mode 100644 index 0000000..f5c5e7a --- /dev/null +++ b/test/fawnds/testConfigs/exp_p16_dis.xml @@ -0,0 +1,14 @@ + + monitor + + partition + 0 + 0 + 16 + + exp_dis.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits4 + diff --git a/test/fawnds/testConfigs/exp_p1_d.xml b/test/fawnds/testConfigs/exp_p1_d.xml new file mode 100644 index 0000000..da23b5b --- /dev/null +++ b/test/fawnds/testConfigs/exp_p1_d.xml @@ -0,0 +1,14 @@ + + monitor + + partition + 0 + 0 + 1 + + exp_d.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits0 + diff --git a/test/fawnds/testConfigs/exp_p1_di.xml b/test/fawnds/testConfigs/exp_p1_di.xml new file mode 100644 index 0000000..93836cb --- /dev/null +++ b/test/fawnds/testConfigs/exp_p1_di.xml @@ -0,0 +1,14 @@ + + monitor + + partition + 0 + 0 + 1 + + exp_di.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits0 + diff --git a/test/fawnds/testConfigs/exp_p1_dis.xml b/test/fawnds/testConfigs/exp_p1_dis.xml new file mode 100644 index 0000000..5c1ce68 --- /dev/null +++ b/test/fawnds/testConfigs/exp_p1_dis.xml @@ -0,0 +1,14 @@ + + monitor + + partition + 0 + 0 + 1 + + exp_dis.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits0 + diff --git a/test/fawnds/testConfigs/exp_p1_ds.xml b/test/fawnds/testConfigs/exp_p1_ds.xml new file mode 100644 index 0000000..1887283 --- /dev/null +++ b/test/fawnds/testConfigs/exp_p1_ds.xml @@ -0,0 +1,14 @@ + + monitor + + partition + 0 + 0 + 1 + + exp_ds.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits0 + diff --git a/test/fawnds/testConfigs/exp_p32_d.xml b/test/fawnds/testConfigs/exp_p32_d.xml new file mode 100644 index 0000000..52cdd45 --- /dev/null +++ b/test/fawnds/testConfigs/exp_p32_d.xml @@ -0,0 +1,14 @@ + + monitor + + partition + 0 + 0 + 32 + + exp_d.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits5 + diff --git a/test/fawnds/testConfigs/exp_p32_di.xml b/test/fawnds/testConfigs/exp_p32_di.xml new file mode 100644 index 0000000..5115dee --- /dev/null +++ b/test/fawnds/testConfigs/exp_p32_di.xml @@ -0,0 +1,14 @@ + + monitor + + partition + 0 + 0 + 32 + + exp_di.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits5 + diff --git a/test/fawnds/testConfigs/exp_p32_dis.xml b/test/fawnds/testConfigs/exp_p32_dis.xml new file mode 100644 index 0000000..7b2eded --- /dev/null +++ b/test/fawnds/testConfigs/exp_p32_dis.xml @@ -0,0 +1,14 @@ + + monitor + + partition + 0 + 0 + 32 + + exp_dis.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits5 + diff --git a/test/fawnds/testConfigs/exp_p32_ds.xml b/test/fawnds/testConfigs/exp_p32_ds.xml new file mode 100644 index 0000000..67f1bd7 --- /dev/null +++ b/test/fawnds/testConfigs/exp_p32_ds.xml @@ -0,0 +1,14 @@ + + monitor + + partition + 0 + 0 + 32 + + exp_ds.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits5 + diff --git a/test/fawnds/testConfigs/exp_p4_d.xml b/test/fawnds/testConfigs/exp_p4_d.xml new file mode 100644 index 0000000..7e60382 --- /dev/null +++ b/test/fawnds/testConfigs/exp_p4_d.xml @@ -0,0 +1,16 @@ + + monitor + + partition + 0 + 0 + 4 + + exp_d.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits2 + + child::store/store1-high-watermark1 + diff --git a/test/fawnds/testConfigs/exp_p4_di.xml b/test/fawnds/testConfigs/exp_p4_di.xml new file mode 100644 index 0000000..95c86d8 --- /dev/null +++ b/test/fawnds/testConfigs/exp_p4_di.xml @@ -0,0 +1,16 @@ + + monitor + + partition + 0 + 0 + 4 + + exp_di.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits2 + + child::store/store1-high-watermark1 + diff --git a/test/fawnds/testConfigs/exp_p4_dis.xml b/test/fawnds/testConfigs/exp_p4_dis.xml new file mode 100644 index 0000000..c1724c5 --- /dev/null +++ b/test/fawnds/testConfigs/exp_p4_dis.xml @@ -0,0 +1,16 @@ + + monitor + + partition + 0 + 0 + 4 + + exp_dis.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits2 + + child::store/store1-high-watermark1 + diff --git a/test/fawnds/testConfigs/exp_p4_dis_h31.xml b/test/fawnds/testConfigs/exp_p4_dis_h31.xml new file mode 100644 index 0000000..0bee8ac --- /dev/null +++ b/test/fawnds/testConfigs/exp_p4_dis_h31.xml @@ -0,0 +1,17 @@ + + monitor + + partition + 0 + 0 + 4 + + exp_dis.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits2 + + child::store/store1-high-watermark32 + child::store/store1-low-watermark31 + diff --git a/test/fawnds/testConfigs/exp_p4_dis_tiny.xml b/test/fawnds/testConfigs/exp_p4_dis_tiny.xml new file mode 100644 index 0000000..f03e4f1 --- /dev/null +++ b/test/fawnds/testConfigs/exp_p4_dis_tiny.xml @@ -0,0 +1,19 @@ + + monitor + + partition + 0 + 0 + 4 + + exp_dis.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits2 + + child::store/store1-high-watermark1 + + child::store/data-len40 + child::store/store2/keys-per-block8 + diff --git a/test/fawnds/testConfigs/exp_p4_ds.xml b/test/fawnds/testConfigs/exp_p4_ds.xml new file mode 100644 index 0000000..7ff9e0b --- /dev/null +++ b/test/fawnds/testConfigs/exp_p4_ds.xml @@ -0,0 +1,16 @@ + + monitor + + partition + 0 + 0 + 4 + + exp_ds.xmlchild::store. + + child::store/store0/hashtable/hash-table-size131072 + child::store/store1/hashtable/hash-table-size131072 + child::store/store2/skip-bits2 + + child::store/store1-high-watermark1 + diff --git a/test/fawnds/testConfigs/testCombi.xml b/test/fawnds/testConfigs/testCombi.xml new file mode 100644 index 0000000..462bfe5 --- /dev/null +++ b/test/fawnds/testConfigs/testCombi.xml @@ -0,0 +1,91 @@ + + monitor + + + + combi + 0 + + 20 + 100 + + + 1000000 + + + sf + + + + ./testFiles/front_file_header + + + cuckoo + + ./testFiles/front_hashtable + 131072 + + + + file + + + 1 + ./testFiles/front_datastore + + + + + sf + + + + ./testFiles/middle_file_header + + + cuckoo + + ./testFiles/middle_hashtable + 131072 + 0 + + + + file + + + 1 + ./testFiles/middle_datastore + + + + + sf_ordered_trie + + + + 1 + + 128 + 0 + + + + file + + + 1 + ./testFiles/back_datastore + + + + + diff --git a/test/fawnds/testConfigs/testCuckoo.xml b/test/fawnds/testConfigs/testCuckoo.xml new file mode 100644 index 0000000..e0eadcd --- /dev/null +++ b/test/fawnds/testConfigs/testCuckoo.xml @@ -0,0 +1,31 @@ + + + sf + 0 + ./testFiles/file_header + + + cuckoo + + + ./testFiles/hashtable + 1048576 + + + + file + + + 1 + ./testFiles/datastore + + + diff --git a/test/fawnds/testConfigs/testFawnDS.xml b/test/fawnds/testConfigs/testFawnDS.xml new file mode 100644 index 0000000..4b63fb0 --- /dev/null +++ b/test/fawnds/testConfigs/testFawnDS.xml @@ -0,0 +1,29 @@ + + monitor + + sf + 0 + ./testFiles/file_header + + + default + + ./testFiles/hashtable + 2097152 + + + + file + + + 1 + ./testFiles/datastore + + + + + diff --git a/test/fawnds/testConfigs/testIterator.xml b/test/fawnds/testConfigs/testIterator.xml new file mode 100644 index 0000000..9be5f9d --- /dev/null +++ b/test/fawnds/testConfigs/testIterator.xml @@ -0,0 +1,21 @@ + + sf + 0 + ./testFiles/file_header + + + default + + + ./testFiles/hashtable + 1048576 + + + + file + + + 1 + ./testFiles/datastore + + diff --git a/test/fawnds/testConfigs/testTrie.xml b/test/fawnds/testConfigs/testTrie.xml new file mode 100644 index 0000000..f10a0ce --- /dev/null +++ b/test/fawnds/testConfigs/testTrie.xml @@ -0,0 +1,18 @@ + + sf_ordered_trie + 0 + 20 + 12 + 1 + 10000 + 512 + 0 + + + file + + + 1 + ./testFiles/datastore + + diff --git a/test/fawnds/testCuckoo.cc b/test/fawnds/testCuckoo.cc new file mode 100644 index 0000000..256e89a --- /dev/null +++ b/test/fawnds/testCuckoo.cc @@ -0,0 +1,179 @@ +#include +#include "fawnds_factory.h" + +#include "gtest/gtest.h" + +#define MAXNUM 262144L +using namespace std; + +namespace fawn +{ + + static std::string conf_file = "testConfigs/testCuckoo.xml"; + uint64_t key_vec1[MAXNUM]; + uint64_t val_vec1[MAXNUM]; + + class FawnDS_Cuckoo_Test : public testing::Test + { + public: + FawnDS *h; + + + protected: + virtual void SetUp() { + // Code here will be called immediately after the constructor (right before + // each test). + srand ( time(NULL) ); + + h = FawnDS_Factory::New(conf_file); // test_num_records_ + FawnDS_Return r = h->Create(); + assert(r == OK); + + uint32_t tag; + char* key; + char* val; + for (uint32_t i = 0; i < MAXNUM ; i++) { + key = (char*) &(key_vec1[i]); + val = (char*) &(val_vec1[i]); + tag = ((rand() % 65535) << 16) + (rand() % 65535); + memcpy(key, &i, 4); + memcpy(key+4, &tag, 4); + val_vec1[i] = rand() % 65535; + //memcpy(val, &i, 4); + } + + } + + virtual void TearDown() { + // Code here will be called immediately after each test (right + // before the destructor). + delete h; + } + + }; + + void test_put_get(uint32_t num, FawnDS *&h, bool stop_if_full = false) { + char *key; + char *val; + uint32_t actual_num = num; + for (uint32_t i = 0; i < num; i++) { + key = (char*) &(key_vec1[i]); + val = (char*) &(val_vec1[i]); + FawnDS_Return ret = h->Put(ConstRefValue(key, 8), ConstRefValue(val, 4)); + if (stop_if_full) { + if (ret == INSUFFICIENT_SPACE) { + actual_num = i; + break; + } + } + else + ASSERT_TRUE(ret == OK); + } + + // should see all keys already inserted + for (uint32_t i = 0; i < actual_num; i++) { + key = (char*) &(key_vec1[i]); + val = (char*) &(val_vec1[i]); + SizedValue<64> read_data; + ASSERT_TRUE(h->Get(ConstRefValue(key, 8), read_data) == OK); + EXPECT_EQ((unsigned) 4, read_data.size()); + EXPECT_EQ(strncmp(val, read_data.data(), 4), 0); + } + + // should not see the key failed in inserting + key = (char*) &(key_vec1[actual_num]); + val = (char*) &(val_vec1[actual_num]); + SizedValue<64> read_data; + ASSERT_TRUE(h->Get(ConstRefValue(key, 8), read_data) == KEY_NOT_FOUND); + + h->Close(); + } + + void test_put_undo_get(FawnDS *&h) { + char *key; + char *val; + long i = 0; + FawnDS_Return r; + for (i = 0; i < MAXNUM; i++) { + key = (char*) &(key_vec1[i]); + val = (char*) &(val_vec1[i]); + r = h->Put(ConstRefValue(key, 8), ConstRefValue(val, 4)); + if (r != OK) + break; + }; + ASSERT_TRUE ( r == INSUFFICIENT_SPACE); + long actual_num = i; + for (i = 0; i < actual_num; i++) { + key = (char*) &(key_vec1[i]); + val = (char*) &(val_vec1[i]); + SizedValue<64> read_data; + ASSERT_TRUE(h->Get(ConstRefValue(key, 8), read_data) == OK); + EXPECT_EQ((unsigned) 4, read_data.size()); + EXPECT_EQ(strncmp(val, read_data.data(), 4), 0); + } + h->Close(); + } + + void test_loadfactor(FawnDS *&h) { + char *key; + char *val; + long count = 0; + long i = 0; + int round = 1; + float average = 0; + int MAXROUND = 10; + while (round <= MAXROUND) { + key = (char*) &(key_vec1[i]); + val = (char*) &(val_vec1[i]); + FawnDS_Return r = h->Put(ConstRefValue(key, 8), ConstRefValue(val, 4)); + if (r == OK) + count ++; + else{ + Value table_size; + h->Status(CAPACITY, table_size); + uint32_t hash_table_size = atoi(table_size.str().c_str()); + printf("trial %d, %ld keys inserted, loadfactor = %.2f%%\n", round, count, 100.0* count/ hash_table_size); + average += 100.0* count/ hash_table_size; + h->Close(); + delete h; + h = FawnDS_Factory::New(conf_file); // test_num_records_ + assert(h); + r = h->Create(); + assert(r == OK); + count = 0; + round ++; + } + i ++; + i = i % MAXNUM; + } + printf("average load factor =%.2f%%\n", average / MAXROUND); + h->Close(); + } + + TEST_F(FawnDS_Cuckoo_Test, SimpleTest_PutGet1Key) { + test_put_get(1, h); + } + + TEST_F(FawnDS_Cuckoo_Test, SimpleTest_PutGet1000Keys) { + test_put_get(1000, h); + } + + TEST_F(FawnDS_Cuckoo_Test, SimpleTest_PutGetManyKeys) { + test_put_get(1000000, h, true); + } + + TEST_F(FawnDS_Cuckoo_Test, SimpleTest_PutUndoGet) { + test_put_undo_get(h); + } + + TEST_F(FawnDS_Cuckoo_Test, SimpleTest_Loadfactor) { + test_loadfactor(h); + } + + +} // namespace + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/fawnds/testFawnDS.cc b/test/fawnds/testFawnDS.cc new file mode 100644 index 0000000..73b0594 --- /dev/null +++ b/test/fawnds/testFawnDS.cc @@ -0,0 +1,383 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include "fawnds_factory.h" +#include "config.h" +#include +#include +#include +#include "print.h" +#include "hashutil.h" + +namespace fawn { + + class FawnDSTest : public testing::Test { + protected: + // Code here will be called immediately after the constructor (right before + // each test). + virtual void SetUp() { + //h = FawnDS_Factory::Create_FawnDS("./testConfigs/testFawnDS.xml"); // test_num_records_ + h = FawnDS_Factory::New("./testConfigs/testFawnDS.xml"); // test_num_records_ + h->Create(); + //h2 = FawnDS_Factory::Create_FawnDS("/localfs/fawn_db2", num_records_); + } + + // Code in the TearDown() method will be called immediately after each test + // (right before the destructor). + virtual void TearDown() { + //EXPECT_EQ(0, unlink("/localfs/fawn_db")); + delete h; + } + + // Objects declared here can be used by all tests in the test case for HashDB. + FawnDS *h; + FawnDS *h2; + + + // private: + // static const uint64_t test_num_records_ = 5000000; + }; + + void test_insert_get(uint32_t order[], uint32_t num, FawnDS *h) { + char key[10]; + char value[4]; + for (uint32_t i = 0; i < num; i++) { + memcpy(key, &(order[0]), 4); + memcpy(key+4, &i, 4); + memcpy(value, &i, 4); + ASSERT_TRUE(h->Put(ConstRefValue(key, 8), ConstRefValue(value, 4)) == OK); + } + for (uint32_t i = 0; i < num; i++) { + memcpy(key, &(order[0]), 4); + memcpy(key+4, &i, 4); + memcpy(value, &i, 4); + SizedValue<64> read_data; + ASSERT_TRUE(h->Get(ConstRefValue(key, 8), read_data) == OK); + EXPECT_EQ(4, read_data.size()); + EXPECT_STREQ(value, read_data.data()); + } + } + + // For the following tests to be a hash table tests, you have to comment any usage of hash functions + // inside the hash table implementation. You also need to set the hash table size to 100. + TEST_F(FawnDSTest, TestCollisionsA) { + uint32_t order[10] = { 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000 }; + test_insert_get(order, 10, h); + } + TEST_F(FawnDSTest, TestCollisionsB) { + uint32_t order[10] = { 100, 100, 100, 500, 500, 500, 100, 100, 200, 200 }; + test_insert_get(order, 10, h); + } + TEST_F(FawnDSTest, TestCollisionsC) { + uint32_t order[10] = { 500, 500, 500, 100, 100, 500, 200, 600, 200, 600 }; + test_insert_get(order, 10, h); + } + + TEST_F(FawnDSTest, TestSimpleInsertRetrieve) { + const char* key = "key0"; + const char* data = "value0"; + ASSERT_TRUE(h->Put(ConstRefValue(key, strlen(key)), ConstRefValue(data, 7)) == OK); + + SizedValue<64> read_data; + ASSERT_TRUE(h->Get(ConstRefValue(key, strlen(key)), read_data) == OK); + EXPECT_EQ(7, read_data.size()); + EXPECT_STREQ(data, read_data.data()); + } + + + TEST_F(FawnDSTest, TestSimpleInsertNovalueRetrieve) { + const char* key = "key0"; + const char* data = ""; + ASSERT_TRUE(h->Put(ConstRefValue(key, strlen(key)), ConstRefValue(data, 0)) == OK); + + SizedValue<64> read_data; + ASSERT_TRUE(h->Get(ConstRefValue(key, strlen(key)), read_data) == OK); + EXPECT_EQ(0, read_data.size()); + //EXPECT_STREQ(data, read_data.data()); + } + + TEST_F(FawnDSTest, TestSimpleDelete) { + //Simple + const char* key = "key0"; + const char* data = "value0"; + //Delete should return false + + ASSERT_FALSE(h->Delete(ConstRefValue(key, strlen(key))) == OK); + ASSERT_TRUE(h->Put(ConstRefValue(key, strlen(key)), ConstRefValue(data, 7)) == OK); + + SizedValue<64> read_data; + ASSERT_TRUE(h->Get(ConstRefValue(key, strlen(key)), read_data) == OK); + EXPECT_EQ(7, read_data.size()); + EXPECT_STREQ(data, read_data.data()); + // Then delete + ASSERT_TRUE(h->Delete(ConstRefValue(key, strlen(key))) == OK); + // Retreive should return false + ASSERT_FALSE(h->Get(ConstRefValue(key, strlen(key)), read_data) == OK); + + const char* key2 = "key1"; + const char* data2 = "value1"; + ASSERT_TRUE(h->Put(ConstRefValue(key2, strlen(key2)), ConstRefValue(data2, 7)) == OK); + ASSERT_TRUE(h->Get(ConstRefValue(key2, strlen(key2)), read_data) == OK); + EXPECT_EQ(7, read_data.size()); + EXPECT_STREQ(data2, read_data.data()); + } + + TEST_F(FawnDSTest, TestDelete) { + + char data[52]; + int* datai = (int*)data; + + for (int i = 0; i < 4; ++i) { + string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + + for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { + datai[j] = i; + } + data[51] = 0; + ASSERT_TRUE(h->Put(ConstRefValue(key), ConstRefValue(data, 52)) == OK); + } + + int i = 0; + string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + + ASSERT_TRUE(h->Delete(ConstRefValue(key)) == OK); + + // Spot open + i = 3; + key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + + for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { + datai[j] = i; + } + data[51] = 0; + ASSERT_TRUE(h->Put(ConstRefValue(key), ConstRefValue(data, 52)) == OK); + + // Spot filled by i=3 + key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + ASSERT_TRUE(h->Delete(ConstRefValue(key)) == OK); + + for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { + datai[j] = i; + } + data[51] = 0; + + SizedValue<64> read_data; + ASSERT_FALSE(h->Get(ConstRefValue(key), read_data) == OK); + } + + + TEST_F(FawnDSTest, Test10000InsertRetrieve) { + char data[52]; + + for (int i = 0; i < 10000; ++i) { + string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + + int* datai = (int*)data; + for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { + datai[j] = i; + } + data[51] = 0; + ASSERT_TRUE(h->Put(ConstRefValue(key), ConstRefValue(data, 52)) == OK); + } + + for (int i = 0; i < 10000; ++i) { + int* datai = (int*)data; + for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { + datai[j] = i; + } + data[51] = 0; + + string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + + SizedValue<64> read_data; + ASSERT_TRUE(h->Get(ConstRefValue(key), read_data) == OK); + EXPECT_EQ(52, read_data.size()); + EXPECT_STREQ(data, read_data.data()); + } + } + + TEST_F(FawnDSTest, Test10000InsertDelete) { + char data[52]; + + for (int i = 0; i < 10000; ++i) { + string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + + int* datai = (int*)data; + for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { + datai[j] = i; + } + data[51] = 0; + ASSERT_TRUE(h->Put(ConstRefValue(key), ConstRefValue(data, 52)) == OK); + } + + for (int i = 0; i < 10000; ++i) { + string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + //fprintf(stderr, "%d\n", i); + ASSERT_TRUE(h->Delete(ConstRefValue(key)) == OK); + } + } + + TEST_F(FawnDSTest, Test10000InsertRewriteRetrieve) { + char data[52]; + + for (int i = 0; i < 10000; ++i) { + string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + int* datai = (int*)data; + for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { + datai[j] = i; + } + data[51] = 0; + ASSERT_TRUE(h->Put(ConstRefValue(key), ConstRefValue(data, 52)) == OK); + } + + // FawnDS* h_new = h->Rewrite(NULL); +// ASSERT_TRUE(h_new != NULL); +// ASSERT_TRUE(h_new->RenameAndClean(h)); +// h = h_new; + +// for (int i = 0; i < 10000; ++i) { +// string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); +// string data_ret; +// int* datai = (int*)data; +// for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { +// datai[j] = i; +// } +// data[51] = 0; +// ASSERT_TRUE(h->Get(key.data(), key.length(), data_ret)); +// EXPECT_EQ(52, data_ret.length()); +// EXPECT_STREQ(data, data_ret.data()); +// } + + } + + + // fawndb should be the same size as previous fawndbs + TEST_F(FawnDSTest, Test10000DoubleInsertRewriteRetrieve) { + char data[52]; + + for (int i = 0; i < 10000; ++i) { + string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + int* datai = (int*)data; + for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { + datai[j] = i; + } + data[51] = 0; + ASSERT_TRUE(h->Put(ConstRefValue(key), ConstRefValue(data, 52)) == OK); + } + + for (int i = 0; i < 10000; ++i) { + string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + int* datai = (int*)data; + for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { + datai[j] = i; + } + data[51] = 0; + ASSERT_TRUE(h->Put(ConstRefValue(key), ConstRefValue(data, 52)) == OK); + } + +// FawnDS* h_new = h->Rewrite(NULL); +// ASSERT_TRUE(h_new != NULL); +// ASSERT_TRUE(h_new->RenameAndClean(h)); +// h = h_new; + +// for (int i = 0; i < 10000; ++i) { +// string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); +// string data_ret; +// int* datai = (int*)data; +// for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { +// datai[j] = i; +// } +// data[51] = 0; +// ASSERT_TRUE(h->Get(key.data(), key.length(), data_ret)); +// EXPECT_EQ(52, data_ret.length()); +// EXPECT_STREQ(data, data_ret.data()); +// } + + } + + TEST_F(FawnDSTest, TestWriteDB) { + char data[1024]; + + // 1GB of values + for (int i = 0; i < 1048576; ++i) { + string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + int* datai = (int*)data; + for (uint j = 0; j < 1024 * sizeof(char) / sizeof(int); ++j) { + datai[j] = i; + } + data[1023] = 0; + //fprintf(stderr, "%d\n", i); + ASSERT_TRUE(h->Put(ConstRefValue(key), ConstRefValue(data, 1024)) == OK); + } + + { + Value status; + if (h->Status(MEMORY_USE, status) == OK) + fprintf(stderr, "Memory use: %s\n", status.str().c_str()); + } + +// // this is required since we're not splitting/merging/rewriting initially +// ASSERT_TRUE(h->WriteHashtableToFile()); + +// // Temporary test of resizing hashtable +// // with current parameters it does not technically resize. +// // this was tested with numentries-deletedentries*5 in resizing calculation. +// FawnDS* h_new = h->Rewrite(NULL); +// ASSERT_TRUE(h_new != NULL); +// ASSERT_TRUE(h_new->RenameAndClean(h)); +// h = h_new; + + } + + TEST_F(FawnDSTest, Test10000Merge) { + char data[52]; + for (int i = 0; i < 5000; ++i) { + string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + int* datai = (int*)data; + for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { + datai[j] = i; + } + data[51] = 0; + ASSERT_TRUE(h->Put(ConstRefValue(key), ConstRefValue(data, 52)) == OK); + } + +// for (int i = 5000; i < 10000; ++i) { +// string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); +// int* datai = (int*)data; +// for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { +// datai[j] = i; +// } +// data[51] = 0; +// ASSERT_TRUE(h2->Insert(key.data(), key.length(), data, 52)); +// } + + +// ASSERT_TRUE(h2->Merge(h, NULL)); + +// for (int i = 0; i < 10000; ++i) { +// string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); +// string data_ret; +// ASSERT_TRUE(h2->Get(key.data(), key.length(), data_ret)); +// } + + } + + TEST_F(FawnDSTest, Test1000000Insert) { + char data[1000]; + + for (int i = 0; i < 1000000; ++i) { + string key = HashUtil::MD5Hash((char*)&i, sizeof(i)); + int* datai = (int*)data; + for (uint j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { + datai[j] = i; + } + data[51] = 0; + ASSERT_TRUE(h->Put(ConstRefValue(key), ConstRefValue(data, 52)) == OK); + } + } + + +} // namespace fawn + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/fawnds/testFiles/.placeholder b/test/fawnds/testFiles/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/test/fawnds/testHash.cc b/test/fawnds/testHash.cc new file mode 100644 index 0000000..19b3f37 --- /dev/null +++ b/test/fawnds/testHash.cc @@ -0,0 +1,51 @@ +#include +#include +#include "fawnds.h" +#include "hash_functions.h" + + + + +using namespace std; +using namespace fawn; + +int main() { + const char* dbname = "fawn_db"; + u_int64_t num_records = 10; + double max_deleted_ratio = .9; + double max_load_factor = .8; + + FawnDS *h; + + cout << "Creating FawnDS" << endl; + h = fawn::FawnDS::Create_FawnDS(dbname, num_records, + max_deleted_ratio, + max_load_factor); + + cout << "Opening FawnDS" << endl; + h = fawn::FawnDS::Open_FawnDS(dbname); + + + const char* key = "key0"; + const char* data = "value0"; + bool irv; + + cout << "Inserting " << key << "," << data << " into " << dbname << endl; + + irv = h->Insert(key, data, 8); + + char** data_returned = (char**) malloc(sizeof(char*)); + u_int64_t* length = (u_int64_t*) malloc(sizeof(u_int64_t)); + bool grv; + + printf("Get value from database\n"); + grv = h->Get(key, data_returned, length); + + if (key != NULL && data != NULL && length != NULL) { + cout << "Key: " << key << " data: " << *data_returned + << " length: " << *length << endl; + } + + + return 0; +} diff --git a/test/fawnds/testIterator.cc b/test/fawnds/testIterator.cc new file mode 100644 index 0000000..864a2ef --- /dev/null +++ b/test/fawnds/testIterator.cc @@ -0,0 +1,129 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ + +#include "fawnds_factory.h" +#include "hashutil.h" +#include "debug.h" + +using namespace fawn; +using namespace std; + +FawnDS* db; + +/* + +void CallForEach(const uint32_t &key_len, const uint32_t &data_len, const char* const &key, const char* const &data, const bool &isDeleted) +{ + printf("Test: CallForEach: %s\n", data); +} + +void CallForEach2(const uint32_t &key_len, const uint32_t &data_len, const char* const &key, const char* const &data, const bool &isDeleted) +{ + char* gData; + uint32_t gData_len; + if (db->Get(key, key_len, gData, gData_len) == 0 && data_len == gData_len && memcmp(data, gData, data_len) == 0) { + printf("Test: CallForEach2: %s\n", data); + } +} + +void CallBeforeRemain() {} +*/ + +int main(int argc, char** argv) +{ + printf("Start testIterator!\n"); + int max = 100; + string config_file = "./testConfigs/testIterator.xml"; + DPRINTF(2, "TestIterator: Will create FawnDS now!\n"); + FawnDS* db = FawnDS_Factory::New(config_file); + db->Create(); + + for(int i = 0; i < max; ++i) + { + uint32_t key_id = HashUtil::FNVHash((const void *)&i, 4); + char key[4]; + memcpy(key, &key_id, 4); + char data[40]; + sprintf(data, "%i %s", i, "aaaa"); + if (db->Put(RefValue(key, 4), RefValue(data, strlen(data)+1)) != 0) { + printf("Insert failed!\n"); + exit(-1); + } else { + printf("Inserted: %s\n", data);// printf("Inserted: %s with key: %s\n", data, key); + } + } + + for(int i = max/4; i < 3*max/4; ++i) + { + uint32_t key_id = HashUtil::FNVHash((const void *)&i, 4); + char key[4]; + memcpy(key, &key_id, 4); + char data[40]; + sprintf(data, "%i %s", i, "bbbb"); + if (db->Put(RefValue(key, 4), RefValue(data, strlen(data)+1)) != 0) { + printf("Insert failed!"); + } else { + printf("Inserted: %s\n", data);// printf("Inserted: %s with key: %s\n", data, key); + } + } + + /* + if(db->DoForEach(CallForEach, CallBeforeRemain) != 0) + printf("DoForEach failed!"); + + if(db->DoForEach(CallForEach2, CallBeforeRemain) != 0) + printf("DoForEach #2 failed!"); + */ + + { + FawnDS_Iterator it = db->Enumerate(); + while (!it.IsEnd()) + { + printf("Test: CallForEach: %s\n", it->data.data()); + ++it; + } + + Value read_data; + it = db->Enumerate(); + while (!it.IsEnd()) + { + if (db->Get(it->key, read_data) == OK && it->data == read_data) { + printf("Test: CallForEach2: %s\n", it->data.data()); + } + ++it; + } + } + + // all iterators need to be destroyed + + /* + auto_ptr it = db->GetIterator(); + while (it->isValid && it->error == 0) { + char print[21]; + int len = 20; + if (it->len < 20) { + len = it->len; + } + memcpy(print, it->data, len); + print[len+1] = '\0'; + printf("Test: CallForEach: %s\n", print); + ++(*it); + } + + it = db->GetIterator(); + while (it->isValid && it->error == 0) { + if (db->IsMostRecentEntry(it->GetIdentifier(), it->fawnds_id, it->data, + it->len) == 0) { + char print[21]; + int len = 20; + if (it->len < 20) { + len = it->len; + } + memcpy(print, it->data, len); + print[len+1] = '\0'; + printf("Test: CallForEach2: %s\n", print); + } + ++(*it); + } + */ + delete db; +} diff --git a/test/fawnds/testTrie.cc b/test/fawnds/testTrie.cc new file mode 100644 index 0000000..be97ba2 --- /dev/null +++ b/test/fawnds/testTrie.cc @@ -0,0 +1,350 @@ +#include "fawnds_factory.h" + +#include +#include +#include + +struct kv_pair { + fawn::Value key; + fawn::Value data; +}; +typedef std::vector kv_array_type; + +static void +generate_random_kv(kv_array_type& out_arr, size_t key_len, size_t data_len, size_t size, unsigned int seed = 0) +{ + size_t kv_len = key_len + data_len; + + srand(seed); + char* buf = new char[kv_len * size]; + for (size_t i = 0; i < kv_len * size; i++) + buf[i] = rand() & 0xff; + //buf[i] = (rand() % ('Z' - 'A')) + 'A'; + + out_arr.clear(); + for (size_t i = 0; i < size; i++) { + kv_pair kv; + kv.key = fawn::RefValue(buf + i * kv_len, key_len); + kv.data = fawn::RefValue(buf + i * kv_len + key_len, data_len); + out_arr.push_back(kv); + } +} + +static void +free_kv(kv_array_type& arr) +{ + char* min = reinterpret_cast(-1); + for (size_t i = 0; i < arr.size(); i++) + if (min > arr[i].key.data()) + min = arr[i].key.data(); + delete [] min; + arr.clear(); +} + +static void +swap(kv_array_type& arr, size_t i, size_t j) +{ + kv_pair t = arr[i]; + arr[i] = arr[j]; + arr[j] = t; +} + +static int +cmp(kv_array_type& arr, size_t key_len, size_t i, size_t j) +{ + return memcmp(arr[i].key.data(), arr[j].key.data(), key_len); +} + +static void +quick_sort(kv_array_type& arr, size_t key_len, size_t left, size_t right) +{ + if (left < right) + { + size_t p = left; + size_t i = left; + size_t j = right + 1; + while (i < j) { + while (i < right && cmp(arr, key_len, ++i, p) <= 0); + while (left < j && cmp(arr, key_len, p, --j) <= 0); + if (i < j) + swap(arr, i, j); + } + swap(arr, p, j); + + if (j > 0) + quick_sort(arr, key_len, left, j - 1); + quick_sort(arr, key_len, j + 1, right); + } +} + +static void +sort_keys(kv_array_type& arr, size_t key_len, size_t off, size_t n) +{ + quick_sort(arr, key_len, off, off + n - 1); +} + +/* +static void +print_hex(const char* s, int len) +{ + for (int i = 0; i < len; i++) + printf("%02x", (unsigned char)s[i]); +} +*/ + +namespace fawn +{ + static std::string conf_file = "testConfigs/testTrie.xml"; + + class FawnDS_SF_Ordered_Trie_Test : public testing::Test + { + protected: + // Code here will be called immediately after the constructor (right before + // each test). + virtual void SetUp() { + Configuration* config = new Configuration(conf_file); + key_len_ = atoi(config->GetStringValue("child::key-len").c_str()); + data_len_ = atoi(config->GetStringValue("child::data-len").c_str()); + size_ = atoi(config->GetStringValue("child::size").c_str()); + assert(key_len_ != 0); + assert(data_len_ != 0); + assert(size_ != 0); + + generate_random_kv(arr_, key_len_, data_len_, size_); + + fawnds_ = FawnDS_Factory::New(config); + fawnds_->Create(); + + ret_data_.resize(0); + } + + // Code in the TearDown() method will be called immediately after each test + // (right before the destructor). + virtual void TearDown() { + free_kv(arr_); + + delete fawnds_; + } + + // Objects declared here can be used by all tests in the test case for HashDB. + + size_t key_len_; + size_t data_len_; + size_t size_; + + kv_array_type arr_; + + FawnDS* fawnds_; + + Value ret_data_; + }; + + TEST_F(FawnDS_SF_Ordered_Trie_Test, TestSimpleInsertRetrieve1) { + EXPECT_EQ(OK, fawnds_->Put(arr_[0].key, arr_[0].data)); + EXPECT_EQ(OK, fawnds_->Flush()); + + EXPECT_EQ(OK, fawnds_->Get(arr_[0].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[0].data.data(), ret_data_.data(), data_len_)); + } + + TEST_F(FawnDS_SF_Ordered_Trie_Test, TestSimpleInsertRetrieve2) { + arr_[0].key.data()[0] = 5; + arr_[1].key.data()[0] = 7; + + EXPECT_EQ(OK, fawnds_->Put(arr_[0].key, arr_[0].data)); + EXPECT_EQ(OK, fawnds_->Put(arr_[1].key, arr_[1].data)); + EXPECT_EQ(OK, fawnds_->Flush()); + + EXPECT_EQ(OK, fawnds_->Get(arr_[0].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[0].data.data(), ret_data_.data(), data_len_)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[1].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[1].data.data(), ret_data_.data(), data_len_)); + } + + TEST_F(FawnDS_SF_Ordered_Trie_Test, TestSimpleUnsortedInsertRetrieve4) { + arr_[0].key.data()[0] = 5; + arr_[1].key.data()[0] = 7; + arr_[2].key.data()[0] = 3; + arr_[3].key.data()[0] = 9; + + EXPECT_EQ(OK, fawnds_->Put(arr_[0].key, arr_[0].data)); + EXPECT_EQ(OK, fawnds_->Put(arr_[1].key, arr_[1].data)); + EXPECT_EQ(INVALID_KEY, fawnds_->Put(arr_[2].key, arr_[2].data)); + EXPECT_EQ(OK, fawnds_->Put(arr_[3].key, arr_[3].data)); + EXPECT_EQ(OK, fawnds_->Flush()); + + EXPECT_EQ(OK, fawnds_->Get(arr_[0].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[0].data.data(), ret_data_.data(), data_len_)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[1].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[1].data.data(), ret_data_.data(), data_len_)); + + EXPECT_EQ(KEY_NOT_FOUND, fawnds_->Get(arr_[2].key, ret_data_)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[3].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[3].data.data(), ret_data_.data(), data_len_)); + } + + TEST_F(FawnDS_SF_Ordered_Trie_Test, TestSimpleSortedInsertRetrieve4) { + arr_[0].key.data()[0] = 3; + arr_[1].key.data()[0] = 5; + arr_[2].key.data()[0] = 7; + arr_[3].key.data()[0] = 9; + + EXPECT_EQ(OK, fawnds_->Put(arr_[0].key, arr_[0].data)); + EXPECT_EQ(OK, fawnds_->Put(arr_[1].key, arr_[1].data)); + EXPECT_EQ(OK, fawnds_->Put(arr_[2].key, arr_[2].data)); + EXPECT_EQ(OK, fawnds_->Put(arr_[3].key, arr_[3].data)); + EXPECT_EQ(OK, fawnds_->Flush()); + + EXPECT_EQ(OK, fawnds_->Get(arr_[0].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[0].data.data(), ret_data_.data(), data_len_)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[1].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[1].data.data(), ret_data_.data(), data_len_)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[2].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[2].data.data(), ret_data_.data(), data_len_)); + + EXPECT_EQ(OK, fawnds_->Get(arr_[3].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[3].data.data(), ret_data_.data(), data_len_)); + } + + TEST_F(FawnDS_SF_Ordered_Trie_Test, TestSimpleSortedInsertRetrieveSome) { + sort_keys(arr_, key_len_, 0, 100); + + for (size_t i = 0; i < 100; i++) + EXPECT_EQ(OK, fawnds_->Put(arr_[i].key, arr_[i].data)); + EXPECT_EQ(OK, fawnds_->Flush()); + + for (size_t i = 0; i < 100; i++) + { + EXPECT_EQ(OK, fawnds_->Get(arr_[i].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[i].data.data(), ret_data_.data(), data_len_)); + } + } + + TEST_F(FawnDS_SF_Ordered_Trie_Test, TestSimpleSortedInsertRetrieveMany) { + sort_keys(arr_, key_len_, 0, size_); + + for (size_t i = 0; i < size_; i++) + EXPECT_EQ(OK, fawnds_->Put(arr_[i].key, arr_[i].data)); + EXPECT_EQ(OK, fawnds_->Flush()); + + { + Value status; + if (fawnds_->Status(MEMORY_USE, status) == OK) + fprintf(stderr, "Memory use: %s\n", status.str().c_str()); + } + + for (size_t i = 0; i < size_; i++) + { + EXPECT_EQ(OK, fawnds_->Get(arr_[i].key, ret_data_)); + EXPECT_EQ(data_len_, ret_data_.size()); + EXPECT_EQ(0, memcmp(arr_[i].data.data(), ret_data_.data(), data_len_)); + } + } + +// TEST_F(FawnDS_SF_Ordered_Trie_Test, TestSimpleSortedInsertRetrieveManyReloaded) { +// sort_keys(arr_, key_len_, 0, size_); +// +// for (size_t i = 0; i < size_; i++) +// EXPECT_EQ(0, fawnds_->Insert(arr_[i], key_len_, arr_[i] + key_len_, data_len_)); +// EXPECT_EQ(0, fawnds_->Flush()); +// +// delete fawnds_; +// fawnds_ = FawnDS_Factory::Open_FawnDS(conf_file); +// +// for (size_t i = 0; i < size_; i++) +// { +// EXPECT_EQ(0, fawnds_->Get(arr_[i], key_len_, ret_data_, ret_data_len_)); +// EXPECT_EQ(data_len_, ret_data_len_); +// EXPECT_EQ(0, memcmp(arr_[i] + key_len_, ret_data_, ret_data_len_)); +// } +// } +// + TEST_F(FawnDS_SF_Ordered_Trie_Test, TestSimpleIterator) { + sort_keys(arr_, key_len_, 0, size_); + + for (size_t i = 0; i < size_; i++) + EXPECT_EQ(OK, fawnds_->Put(arr_[i].key, arr_[i].data)); + EXPECT_EQ(OK, fawnds_->Flush()); + + size_t idx = 0; + for (FawnDS_ConstIterator it = fawnds_->Enumerate(); !it.IsEnd(); ++it) + { + EXPECT_EQ(key_len_, it->key.size()); + EXPECT_EQ(data_len_, it->data.size()); + if (idx < size_) { + EXPECT_EQ(0, memcmp(arr_[idx].key.data(), it->key.data(), key_len_)); + EXPECT_EQ(0, memcmp(arr_[idx].data.data(), it->data.data(), data_len_)); + } + idx++; + } + EXPECT_EQ(size_, idx); + } + + TEST_F(FawnDS_SF_Ordered_Trie_Test, TestGetBeforeFinalizing) { + sort_keys(arr_, key_len_, 0, size_); + + EXPECT_EQ(OK, fawnds_->Put(arr_[0].key, arr_[0].data)); + EXPECT_EQ(ERROR, fawnds_->Get(arr_[0].key, ret_data_)); + } + + TEST_F(FawnDS_SF_Ordered_Trie_Test, TestInsertAfterFinalizing) { + sort_keys(arr_, key_len_, 0, size_); + + EXPECT_EQ(OK, fawnds_->Flush()); + EXPECT_EQ(ERROR, fawnds_->Put(arr_[0].key, arr_[0].data)); + } + + TEST_F(FawnDS_SF_Ordered_Trie_Test, TestFlushAfterFinalizing) { + sort_keys(arr_, key_len_, 0, size_); + + EXPECT_EQ(OK, fawnds_->Flush()); + EXPECT_EQ(OK, fawnds_->Flush()); + } + +// TEST_F(FawnDS_SF_Ordered_Trie_Test, TestInsertAfterReloading) { +// sort_keys(arr_, key_len_, 0, size_); +// +// EXPECT_EQ(0, fawnds_->Insert(arr_[0], key_len_, arr_[0] + key_len_, data_len_)); +// EXPECT_EQ(0, fawnds_->Flush()); +// +// delete fawnds_; +// fawnds_ = FawnDS_Factory::Open_FawnDS(conf_file); +// +// EXPECT_EQ(-1, fawnds_->Insert(arr_[1], key_len_, arr_[1] + key_len_, data_len_)); +// } +// +// TEST_F(FawnDS_SF_Ordered_Trie_Test, TestFlushAfterReloading) { +// sort_keys(arr_, key_len_, 0, size_); +// +// EXPECT_EQ(0, fawnds_->Insert(arr_[0], key_len_, arr_[0] + key_len_, data_len_)); +// EXPECT_EQ(0, fawnds_->Flush()); +// +// delete fawnds_; +// fawnds_ = FawnDS_Factory::Open_FawnDS(conf_file); +// +// EXPECT_EQ(-1, fawnds_->Flush()); +// } + +} // namespace fawn + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/test/fawnds/testWorkloads/README b/test/fawnds/testWorkloads/README new file mode 100644 index 0000000..56de193 --- /dev/null +++ b/test/fawnds/testWorkloads/README @@ -0,0 +1,15 @@ +=================================================== +to generate our predefined workload trace: + +Step0, install YCSB, go to your YCSB root dir + +Step1, run ./workload_gen.sh [path-to-YSCB] + +=================================================== +useful options to customize workloads: + +{insert,read,update,scan}proportion -- the portion to send given type of queires +operationcount -- the number of operations +fieldcount -- the number of fields in the table. For our K-V application, we can simply set this option to 1 (just a "value" field) +recordcount -- the number of different keys inserted or read. In YCSB, keys will look like "user1941507751" +fieldlength -- the length of value when insert to the DB diff --git a/test/fawnds/testWorkloads/common.sh b/test/fawnds/testWorkloads/common.sh new file mode 100644 index 0000000..5971cf9 --- /dev/null +++ b/test/fawnds/testWorkloads/common.sh @@ -0,0 +1,41 @@ + +# just to check the format/validity of the source YCSB data +function generate_load_raw { + # $1 = filename prefix for a "load" trace + # $2 = (UNUSED) + # $3 = arguments to YCSB + + echo generating $1.load.raw + + java -cp ${YCSB_DIR}/build/ycsb.jar com.yahoo.ycsb.Client -load -db com.yahoo.ycsb.BasicDB $3 -s > $1.load.raw +} + +function generate_load { + # $1 = filename prefix for a "load" trace + # $2 = arguments to preprocessTrace + # $3 = arguments to YCSB + + echo generating $1.load + + java -cp ${YCSB_DIR}/build/ycsb.jar com.yahoo.ycsb.Client -load -db com.yahoo.ycsb.BasicDB $3 -s | ../preprocessTrace $2 $1.load +} + +function generate_trans { + # $1 = filename prefix for a "transaction" trace + # $2 = arguments to preprocessTrace + # $3 = arguments to YCSB + + echo generating $1.trans + + java -cp ${YCSB_DIR}/build/ycsb.jar com.yahoo.ycsb.Client -t -db com.yahoo.ycsb.BasicDB $3 -s | ../preprocessTrace $2 $1.trans +} + +if [ $# -ne 1 ] +then + echo "Usage: $0 [path-to-YSCB]" + echo " e.g. $0 /Users/binfan/projects/fawn/YCSB" + exit 1 +fi + +YCSB_DIR=$1 + diff --git a/test/fawnds/testWorkloads/exp_0GB b/test/fawnds/testWorkloads/exp_0GB new file mode 100644 index 0000000..1e22e91 --- /dev/null +++ b/test/fawnds/testWorkloads/exp_0GB @@ -0,0 +1,12 @@ +workload=com.yahoo.ycsb.workloads.CoreWorkload + +fieldcount=1 +# actual fieldlength will be specified by preprocessTrace +fieldlength=0 +readallfields=true + +# total 0 GB for fieldlength=1000 +recordcount=1 + +# without this, YCSB may generate duplicate insert entries +insertorder=ordered diff --git a/test/fawnds/testWorkloads/exp_100GB b/test/fawnds/testWorkloads/exp_100GB new file mode 100644 index 0000000..55f687f --- /dev/null +++ b/test/fawnds/testWorkloads/exp_100GB @@ -0,0 +1,12 @@ +workload=com.yahoo.ycsb.workloads.CoreWorkload + +fieldcount=1 +# actual fieldlength will be specified by preprocessTrace +fieldlength=0 +readallfields=true + +# total 100 GB for fieldlength=1000 +recordcount=100000000 + +# without this, YCSB may generate duplicate insert entries +insertorder=ordered diff --git a/test/fawnds/testWorkloads/exp_50GB b/test/fawnds/testWorkloads/exp_50GB new file mode 100644 index 0000000..38ab6ee --- /dev/null +++ b/test/fawnds/testWorkloads/exp_50GB @@ -0,0 +1,12 @@ +workload=com.yahoo.ycsb.workloads.CoreWorkload + +fieldcount=1 +# actual fieldlength will be specified by preprocessTrace +fieldlength=0 +readallfields=true + +# total 50 GB for fieldlength=1000 +recordcount=50000000 + +# without this, YCSB may generate duplicate insert entries +insertorder=ordered diff --git a/test/fawnds/testWorkloads/exp_75GB b/test/fawnds/testWorkloads/exp_75GB new file mode 100644 index 0000000..05645ce --- /dev/null +++ b/test/fawnds/testWorkloads/exp_75GB @@ -0,0 +1,12 @@ +workload=com.yahoo.ycsb.workloads.CoreWorkload + +fieldcount=1 +# actual fieldlength will be specified by preprocessTrace +fieldlength=0 +readallfields=true + +# total 75 GB for fieldlength=1000 +recordcount=75000000 + +# without this, YCSB may generate duplicate insert entries +insertorder=ordered diff --git a/test/fawnds/testWorkloads/exp_empty b/test/fawnds/testWorkloads/exp_empty new file mode 100644 index 0000000..a4a017a --- /dev/null +++ b/test/fawnds/testWorkloads/exp_empty @@ -0,0 +1,6 @@ +operationcount=1 +readproportion=1 +updateproportion=0 +scanproportion=0 +insertproportion=0 +requestdistribution=uniform diff --git a/test/fawnds/testWorkloads/exp_insert_perf_10_250M b/test/fawnds/testWorkloads/exp_insert_perf_10_250M new file mode 100644 index 0000000..6c4b55f --- /dev/null +++ b/test/fawnds/testWorkloads/exp_insert_perf_10_250M @@ -0,0 +1,7 @@ +operationcount=250000000 +readproportion=0.90 +updateproportion=0 +scanproportion=0 +insertproportion=0.10 +requestdistribution=uniform + diff --git a/test/fawnds/testWorkloads/exp_insert_perf_10_500M b/test/fawnds/testWorkloads/exp_insert_perf_10_500M new file mode 100644 index 0000000..589499e --- /dev/null +++ b/test/fawnds/testWorkloads/exp_insert_perf_10_500M @@ -0,0 +1,7 @@ +operationcount=500000000 +readproportion=0.90 +updateproportion=0 +scanproportion=0 +insertproportion=0.10 +requestdistribution=uniform + diff --git a/test/fawnds/testWorkloads/exp_lookup_perf b/test/fawnds/testWorkloads/exp_lookup_perf new file mode 100644 index 0000000..2293709 --- /dev/null +++ b/test/fawnds/testWorkloads/exp_lookup_perf @@ -0,0 +1,7 @@ +# 47.6 min at 35 K QPS +operationcount=100000000 +readproportion=1 +updateproportion=0 +scanproportion=0 +insertproportion=0 +requestdistribution=uniform diff --git a/test/fawnds/testWorkloads/exp_tiny_10GB b/test/fawnds/testWorkloads/exp_tiny_10GB new file mode 100644 index 0000000..3273b9b --- /dev/null +++ b/test/fawnds/testWorkloads/exp_tiny_10GB @@ -0,0 +1,12 @@ +workload=com.yahoo.ycsb.workloads.CoreWorkload + +fieldcount=1 +# actual fieldlength will be specified by preprocessTrace +fieldlength=0 +readallfields=true + +# total 10 GB for fieldlength=40 (+ 24 other data) +recordcount=156250000 + +# without this, YCSB may generate duplicate insert entries +insertorder=ordered diff --git a/test/fawnds/testWorkloads/exp_tiny_1GB b/test/fawnds/testWorkloads/exp_tiny_1GB new file mode 100644 index 0000000..224b6e8 --- /dev/null +++ b/test/fawnds/testWorkloads/exp_tiny_1GB @@ -0,0 +1,12 @@ +workload=com.yahoo.ycsb.workloads.CoreWorkload + +fieldcount=1 +# actual fieldlength will be specified by preprocessTrace +fieldlength=0 +readallfields=true + +# total 1 GB for fieldlength=40 (+ 24 other data) +recordcount=15625000 + +# without this, YCSB may generate duplicate insert entries +insertorder=ordered diff --git a/test/fawnds/testWorkloads/exp_tiny_6.4GB b/test/fawnds/testWorkloads/exp_tiny_6.4GB new file mode 100644 index 0000000..f362cc4 --- /dev/null +++ b/test/fawnds/testWorkloads/exp_tiny_6.4GB @@ -0,0 +1,12 @@ +workload=com.yahoo.ycsb.workloads.CoreWorkload + +fieldcount=1 +# actual fieldlength will be specified by preprocessTrace +fieldlength=0 +readallfields=true + +# total 6.4 GB for fieldlength=40 (+ 20 B key + 4 B metadata) +recordcount=100000000 + +# without this, YCSB may generate duplicate insert entries +insertorder=ordered diff --git a/test/fawnds/testWorkloads/exp_update_perf_10 b/test/fawnds/testWorkloads/exp_update_perf_10 new file mode 100644 index 0000000..f985e6c --- /dev/null +++ b/test/fawnds/testWorkloads/exp_update_perf_10 @@ -0,0 +1,7 @@ +# 2.5 hours at 22.22 K QPS +operationcount=200000000 +readproportion=0.90 +updateproportion=0.10 +scanproportion=0 +insertproportion=0 +requestdistribution=uniform diff --git a/test/fawnds/testWorkloads/exp_update_perf_50 b/test/fawnds/testWorkloads/exp_update_perf_50 new file mode 100644 index 0000000..5a3b676 --- /dev/null +++ b/test/fawnds/testWorkloads/exp_update_perf_50 @@ -0,0 +1,7 @@ +# 0.56 hours at 50 K QPS +operationcount=100000000 +readproportion=0.50 +updateproportion=0.50 +scanproportion=0 +insertproportion=0 +requestdistribution=uniform diff --git a/test/fawnds/testWorkloads/gen_background_operations.sh b/test/fawnds/testWorkloads/gen_background_operations.sh new file mode 100755 index 0000000..be06eae --- /dev/null +++ b/test/fawnds/testWorkloads/gen_background_operations.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +source common.sh + +## for background operation impacts +generate_load exp_100GB 1000 "-P exp_100GB" & +generate_trans exp_100GB_update_perf_10 1000 "-P exp_100GB -P exp_update_perf_10" & + +# or smaller workload +#generate_load exp_50GB 1000 "-P exp_50GB" & +#generate_trans exp_50GB_update_perf_10 1000 "-P exp_50GB -P exp_update_perf_10" & + +wait + +echo done + diff --git a/test/fawnds/testWorkloads/gen_get_latency.sh b/test/fawnds/testWorkloads/gen_get_latency.sh new file mode 100755 index 0000000..8b64381 --- /dev/null +++ b/test/fawnds/testWorkloads/gen_get_latency.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +source common.sh + +## for GET latency and GET latency breakdown +generate_load exp_100GB 1000 "-P exp_100GB" & +generate_trans exp_100GB_lookup_perf 1000 "-P exp_100GB -P exp_lookup_perf" & + +wait + +echo done + diff --git a/test/fawnds/testWorkloads/gen_index_size_comparison.sh b/test/fawnds/testWorkloads/gen_index_size_comparison.sh new file mode 100755 index 0000000..5329980 --- /dev/null +++ b/test/fawnds/testWorkloads/gen_index_size_comparison.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +source common.sh + +## for index size change graph +generate_load exp_50GB 1000 "-P exp_50GB" & +generate_trans exp_50GB_insert_perf_10 1000 "-P exp_50GB -P exp_insert_perf_10_500M" & + +wait + +echo done + diff --git a/test/fawnds/testWorkloads/gen_tiny.sh b/test/fawnds/testWorkloads/gen_tiny.sh new file mode 100755 index 0000000..9cf0298 --- /dev/null +++ b/test/fawnds/testWorkloads/gen_tiny.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +source common.sh + +## for tiny data +generate_load exp_tiny_6.4GB 40 "-P exp_tiny_6.4GB" & +generate_trans exp_tiny_6.4GB_update_perf_50 40 "-P exp_tiny_6.4GB -P exp_update_perf_50" & + +wait + +echo done + diff --git a/test/fawnds/tmp/.placeholder b/test/fawnds/tmp/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/utils/Makefile.am b/utils/Makefile.am new file mode 100644 index 0000000..6b2c85f --- /dev/null +++ b/utils/Makefile.am @@ -0,0 +1,3 @@ +noinst_LTLIBRARIES = libfawnkvutils.la +noinst_HEADERS = hashutil.h dbid.h print.h fnv.h datastat.h +libfawnkvutils_la_SOURCES = hashutil.cc dbid.cc print.cc timing.c datastat.cc diff --git a/utils/Makefile.in b/utils/Makefile.in new file mode 100644 index 0000000..54bcfaf --- /dev/null +++ b/utils/Makefile.in @@ -0,0 +1,510 @@ +# Makefile.in generated by automake 1.11.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +# 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, +# Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = utils +DIST_COMMON = $(noinst_HEADERS) $(srcdir)/Makefile.am \ + $(srcdir)/Makefile.in +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/libtool.m4 \ + $(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \ + $(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \ + $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libfawnkvutils_la_LIBADD = +am_libfawnkvutils_la_OBJECTS = hashutil.lo dbid.lo print.lo timing.lo \ + datastat.lo +libfawnkvutils_la_OBJECTS = $(am_libfawnkvutils_la_OBJECTS) +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__depfiles_maybe = depfiles +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +CCLD = $(CC) +LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +LTCXXCOMPILE = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +CXXLD = $(CXX) +CXXLINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=link $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +SOURCES = $(libfawnkvutils_la_SOURCES) +DIST_SOURCES = $(libfawnkvutils_la_SOURCES) +HEADERS = $(noinst_HEADERS) +ETAGS = etags +CTAGS = ctags +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +MAKEINFO = @MAKEINFO@ +MKDIR_P = @MKDIR_P@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +RANLIB = @RANLIB@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +VERSION = @VERSION@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +lt_ECHO = @lt_ECHO@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +sysconfdir = @sysconfdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libfawnkvutils.la +noinst_HEADERS = hashutil.h dbid.h print.h fnv.h datastat.h +libfawnkvutils_la_SOURCES = hashutil.cc dbid.cc print.cc timing.c datastat.cc +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .cc .lo .o .obj +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu utils/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --gnu utils/Makefile +.PRECIOUS: Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" != "$$p" || dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done +libfawnkvutils.la: $(libfawnkvutils_la_OBJECTS) $(libfawnkvutils_la_DEPENDENCIES) + $(CXXLINK) $(libfawnkvutils_la_OBJECTS) $(libfawnkvutils_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/datastat.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbid.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hashutil.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/print.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/timing.Plo@am__quote@ + +.c.o: +@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(COMPILE) -c $< + +.c.obj: +@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(COMPILE) -c `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LTCOMPILE) -c -o $@ $< + +.cc.o: +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ $< + +.cc.obj: +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.cc.lo: +@am__fastdepCXX_TRUE@ $(LTCXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + mkid -fID $$unique +tags: TAGS + +TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + set x; \ + here=`pwd`; \ + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: CTAGS +CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + `test -z '$(STRIP)' || \ + echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \ + clean-libtool clean-noinstLTLIBRARIES ctags distclean \ + distclean-compile distclean-generic distclean-libtool \ + distclean-tags distdir dvi dvi-am html html-am info info-am \ + install install-am install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags uninstall uninstall-am + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/utils/datastat.cc b/utils/datastat.cc new file mode 100644 index 0000000..021a003 --- /dev/null +++ b/utils/datastat.cc @@ -0,0 +1,162 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include +#include +#include +#include +#include + +#include "datastat.h" +namespace fawn { + DataStat::DataStat(const char* p, double v_min, double v_max, + double step_linear = 1.0f, + double start_log = 1.0f, + double step_log = 10.0f) + : v_min(v_min), v_max(v_max), + step_linear(step_linear), start_log(start_log), step_log(step_log), + x_min(v_max), x_max(v_min), x_num(0), x_sum(0.0f) + { + assert(v_min <= v_max); + assert(step_linear > 0); + assert(start_log > 0); + assert(step_log > 1); + + memset(name, 0, 256); + strncpy(name, p, 256); + + num_linear = (size_t) ceil((v_max - v_min) / step_linear); + num_log = (size_t) 1 + ceil( log(v_max / start_log) / log(step_log)); + count_linear = new size_t[num_linear + 1]; + count_log = new size_t[num_log + 1]; + memset(count_linear, 0, sizeof(size_t) * num_linear); + memset(count_log, 0, sizeof(size_t) * num_log); + } + + DataStat::~DataStat() + { + } + + + size_t DataStat::index_linear(double x) { + size_t i; + i = (size_t) (x - v_min) / step_linear; + i = ( i > 0 ) ? i : 0; + i = ( i < num_linear ) ? i : num_linear - 1; + return i; + } + + size_t DataStat::index_log(double x) { + size_t i; + if (x < start_log) { + i = 0; + } else { + i = (size_t) 1 + log(x / start_log) / log(step_log); + } + i = ( i > 0 ) ? i : 0; + i = ( i < num_log ) ? i : num_log - 1; + return i; + } + + void DataStat::insert(double x) { + x_num ++; + x_sum += x; + x_max = (x_max > x ) ? x_max : x; + x_min = (x_min < x ) ? x_min : x; + count_linear[index_linear(x)] ++; + count_log[index_log(x)] ++; + } + + double DataStat::percentile(double pct) { + int i; + double th = x_num * pct; + size_t cur; + double ret_linear = v_min - 1; + cur = 0; + for (i = 0; i < num_linear; i++) { + cur += count_linear[i]; + if (cur >= th ) { + ret_linear = v_min + i * step_linear; + break; + } + } + + double ret_log = v_min - 1; + cur = 0; + for (i = 0; i < num_log; i++) { + cur += count_log[i]; + if (cur >= th ) { + if ( i == 0 ) + ret_log = v_min; + else + ret_log = start_log * pow(step_log, i - 1); + break; + } + } + + return std::max(ret_linear, ret_log); + } + + double DataStat::cumulative(double x) { + double cum_linear = 0; + for (int i = 0; i < index_linear(x); i++) + cum_linear += count_linear[i]; + cum_linear = cum_linear * 1.0 / num(); + + double cum_log = 0; + for (int i = 0; i < index_log(x); i++) + cum_log += count_log[i]; + cum_log = cum_log * 1.0 / num(); + + return std::max(cum_linear, cum_log); + } + + void DataStat::summary() { + printf("========================================\n"); + printf("\tsummary of %s\n", name); + printf("----------------------------------------\n"); + if (x_num) { + printf("Number %llu\n", static_cast(num())); + printf("Average %f\n", mean()); + printf("Median %f\n", percentile(0.5)); + printf("Min %f\n", x_min); + printf("95%%Min %f\n", percentile(0.05)); + printf("99%%Min %f\n", percentile(0.01)); + printf("99.9%%Min %f\n", percentile(0.001)); + printf("Max %f\n", x_max); + printf("95%%Max %f\n", percentile(0.95)); + printf("99%%Max %f\n", percentile(0.99)); + printf("99.9%%Max %f\n", percentile(0.999)); + } else + printf("N/A\n"); + printf("========================================\n"); + } + + void DataStat::cdf(double intv, bool logscale = false) { + if (logscale) + assert(intv > 1); + printf("========================================\n"); + printf("\tCDF of %s\n", name); + printf("----------------------------------------\n"); + if (num() == 0) { + printf("N/A\n"); + printf("========================================\n"); + return; + } + + double x; + if (logscale) { + x = start_log; + } + else { + x = v_min; + } + while (x < v_max) { + printf("%f\t%f\n", x, cumulative(x)); + if (logscale) + x *= intv; + else + x += intv; + } + + printf("========================================\n"); + } +} diff --git a/utils/datastat.h b/utils/datastat.h new file mode 100644 index 0000000..a6ff1c1 --- /dev/null +++ b/utils/datastat.h @@ -0,0 +1,62 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _STATUTIL_H_ +#define _STATUTIL_H_ + +//#define MAX_BUCKETS 1000 + + +using namespace std; +namespace fawn { + class DataStat { + public: + DataStat(const char* p, double v_min, double v_max, + double step_linear, + double start_log, double step_log); + ~DataStat(); + /* insert a new value x */ + void insert(double x); + + + size_t index_linear(double x); + size_t index_log(double x); + + + /* return the value of given percentile */ + double percentile(double pct); + /* return the cumulative probability of value x */ + double cumulative(double x); + /* return the max value */ + double max() {return x_max;} + /* return the min value */ + double min() {return x_min;} + /* return the sum of all values */ + double sum() {return x_sum;} + /* return the mean value */ + double mean() {return x_sum/x_num;} + /* return the num of values tracked */ + size_t num() {return x_num;} + /* return the median value, i.e. 50% percentile*/ + double median() {return percentile(0.5);} + /* print a brief summary of this data series */ + void summary(); + /* print the cdf of this data series */ + void cdf(double intv, bool logscale); + + + private: + char name[256]; + double v_min, v_max; // max and min of possible values + double x_min, x_max; // max and min of input data + double x_sum; // sum of all input data + size_t x_num; // num of all input data + + size_t num_linear; + size_t num_log; + size_t *count_linear; + size_t *count_log; + double step_linear; + double start_log; + double step_log; + }; +} +#endif /* _STATUTIL_H_ */ diff --git a/utils/dbid.cc b/utils/dbid.cc new file mode 100644 index 0000000..5de38f5 --- /dev/null +++ b/utils/dbid.cc @@ -0,0 +1,149 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +using namespace std; +#include +#include +#include "dbid.h" +#include "print.h" + + +namespace fawn { + DBID::DBID() : actual_size(0) { + memset(value, '\0', sizeof(value)); + } + + DBID::DBID(const char* c, unsigned int size) { + Init(c, size); + } + + DBID::DBID(const string &s) { + Init(s.data(), s.size()); + } + + void DBID::Init(const char *c, unsigned int size) { + // Keep track of input size (vs. DBID_LENGTH) + actual_size = size; + + if (size > DBID_LENGTH) { + cout << "Only " << DBID_LENGTH << " byte IDs supported, size = " << size << ". Truncating ..." << endl; + cout << "Change DBID_LENGTH in dbid.h to a large number to support larger key sizes." << endl; + actual_size = size = DBID_LENGTH; + } + + // Set value + memcpy(value, c, size); + + // Pad entry with 0s if needed + if (size < DBID_LENGTH) + memset(value+size, '\0', DBID_LENGTH - size); + + } + + DBID::~DBID() { + + } + + string* DBID::actual_data() { + return new string((char *)value, actual_size); + } + + const string DBID::actual_data_str() const { + return string((char *)value, actual_size); + } + + + void DBID::printValue() const { + cout << "actual dbid size: " << actual_size << endl; + print_payload((const u_char*)value, DBID_LENGTH); + } + + void DBID::printValue(int prefix_bytes) const { + cout << "actual dbid size: " << actual_size << ", printing first " << prefix_bytes << " bytes of dbid" << endl; + print_payload((const u_char*)value, prefix_bytes); + } + + + bool DBID::operator==(const DBID &rhs) const { + return (memcmp(value, rhs.value, DBID_LENGTH) == 0); + } + + // Bytewise comparison of equal-length byte arrays + bool DBID::operator<(const DBID &rhs) const { + for (uint i = 0; i < DBID_LENGTH; i++) { + if ((unsigned int)value[i] > (unsigned int)rhs.value[i]) { + return false; + } + if ((unsigned int)value[i] < (unsigned int)rhs.value[i]) { + return true; + } + } + return false; + } + + uint64_t DBID::operator-(const DBID &rhs) const { + uint64_t thisid = (uint64_t) ((0xff & value[0]) + + ((0xff & value[1]) << 8) + + ((0xff & value[2]) << 16) + + ((0xff & value[3]) << 24) + + ((uint64_t)(0xff & value[4]) << 32) + + ((uint64_t)(0xff & value[5]) << 40) + + ((uint64_t)(0xff & value[6]) << 48) + + ((uint64_t)(0xff & value[7]) << 56)); + + uint64_t rhsid = (uint64_t) ((0xff & rhs.value[0]) + + ((0xff & rhs.value[1]) << 8) + + ((0xff & rhs.value[2]) << 16) + + ((0xff & rhs.value[3]) << 24) + + ((uint64_t)(0xff & rhs.value[4]) << 32) + + ((uint64_t)(0xff & rhs.value[5]) << 40) + + ((uint64_t)(0xff & rhs.value[6]) << 48) + + ((uint64_t)(0xff & rhs.value[7]) << 56)); + if (rhsid > thisid) + return rhsid - thisid; + else + return thisid - rhsid; + } + + // returns this - rhs + // DBID DBID::operator-(const DBID &rhs) const + // { + // int8_t carryover = 0; + // char ans[DBID_LENGTH]; + // for (uint i = DBID_LENGTH-1; i >= 0; i--) { + // uint8_t diff = ((value[i] & 0xff) - (rhs.value[i] & 0xff)) & 0xff; + // carryover = (diff == 0 && carryover == 1); + // ans[i] = (diff - carryover) & 0xff; + // } + // return DBID(ans, DBID_LENGTH); + // } + + + + // Comparison function for ring space. + bool between(const string& startKey, const string& endKey, const string& thisKey) { + DBID startID(startKey); + DBID endID(endKey); + DBID thisID(thisKey); + + return between(&startID, &endID, &thisID); + } + + bool between(const DBID* startID, const DBID* endID, const DBID* thisID) { + if (*startID == *endID) { + return true; + } else if (*endID < *startID) { + // endKey < startKey, so there is wrap around + if (*startID < *thisID || !(*endID < *thisID)) { + return true; + } + } else { + // startKey < endKey + // if startKey < key and !(key > endKey) + if (*startID < *thisID && !(*endID < *thisID)) { + return true; + } + } + + return false; + } + +} diff --git a/utils/dbid.h b/utils/dbid.h new file mode 100644 index 0000000..40f1a7f --- /dev/null +++ b/utils/dbid.h @@ -0,0 +1,72 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _DBID_H_ +#define _DBID_H_ + +using namespace std; +#include +#include +#include + +/** + * DBID is a fixed-length identifier for nodes on a ringspace. Since + * it is fixed length, the length needs to be as long as the longest + * allowable key size so that the ID space contains the granularity to + * load balance data. This length is defined as DBID_LENGTH below. + * + * In other words: DBID_LENGTH is the maximum key length for any query. + * + * Comparisons of short keys will pad the incoming key with 0s to + * meet this size when necessary. You can redefine DBID_LENGTH to + * whatever size you desire, but DBID comparisons will take time + * proportional to the size. + **/ + +namespace fawn { + const uint32_t DBID_LENGTH = 1024; + + class DBID + { + + private: + char value[DBID_LENGTH]; + unsigned int actual_size; + public: + DBID(); + DBID(const char *c, unsigned int size); + explicit DBID(const string &s); + ~DBID(); + void Init(const char *c, unsigned int size); + + char* data() { + return value; + } + const char *const_data() const { + return (const char *)value; + } + string* actual_data(); + const string actual_data_str() const; + uint32_t size() { + return DBID_LENGTH; + } + unsigned int get_actual_size() { + return actual_size; + } + void printValue() const; + void printValue(int prefix_bytes) const; + void set_actual_size(unsigned int as) { + actual_size = as; + } + + bool operator==(const DBID &rhs) const; + bool operator<(const DBID &rhs) const; + + // This compares only the top 64 bits of each DBID + uint64_t operator-(const DBID &rhs) const; + + //DBID operator-(const DBID &rhs) const; + }; + + bool between(const string& startKey, const string& endKey, const string& thisKey); + bool between(const DBID* startID, const DBID* endID, const DBID* thisID); +} +#endif diff --git a/utils/debug.c b/utils/debug.c new file mode 100644 index 0000000..67489c8 --- /dev/null +++ b/utils/debug.c @@ -0,0 +1,59 @@ +/* + * Rather boring. Define the debugging stuff. + */ + +#include +#include +#include +#include +//for time +#include +#include + +#include "debug.h" + + +unsigned int debug = 0; + +/* We could autogenerate this if we felt like it */ + +struct debug_def { + int debug_val; + char *debug_def; +}; + +static +struct debug_def debugs[] = { +#include "debug-text.h" + { 0, NULL } /* End of list marker */ +}; + +int set_debug(char *arg) +{ + int i; + if (!arg || arg[0] == '\0') { + return -1; + } + + if (arg[0] == '?' || !strcmp(arg, "list")) { + fprintf(stderr, + "Debug values and definitions\n" + "----------------------------\n"); + for (i = 0; debugs[i].debug_def != NULL; i++) { + fprintf(stderr, "%5d %s\n", debugs[i].debug_val, + debugs[i].debug_def); + } + fprintf(stderr, "\n\'all\' will enable all debug flags.\n"); + return -1; + } + if (!strcmp(arg, "all")) { + debug = 0xffffffff; + return 0; + } + + if (isdigit(arg[0])) { + debug |= atoi(arg); + } + return 0; +} + diff --git a/utils/debug.h b/utils/debug.h new file mode 100644 index 0000000..567ee44 --- /dev/null +++ b/utils/debug.h @@ -0,0 +1,45 @@ +#ifndef _DEBUG_H_ +#define _DEBUG_H_ + +#include /* for perror */ +#define eprintf(fmt, args...) fprintf(stderr, fmt, ##args) + +#ifndef DEBUG +//#define DEBUG +#endif + +#ifdef DEBUG +//extern unsigned int debug; + +#define debug_level (1 | 2) + +#define DPRINTF(level, fmt, args...) \ + do { if (debug_level & (level)) fprintf(stderr, fmt , ##args ); } while(0) +#define DEBUG_PERROR(errmsg) \ + do { if (debug_level & DEBUG_ERRS) perror(errmsg); } while(0) +#else +#define DPRINTF(args...) +#define DEBUG_PERROR(args...) +#endif + +/* + * The format of this should be obvious. Please add some explanatory + * text if you add a debugging value. This text will show up in + * -d list + */ +#define DEBUG_NONE 0x00 // DBTEXT: No debugging +#define DEBUG_ERRS 0x01 // DBTEXT: Verbose error reporting +#define DEBUG_FLOW 0x02 // DBTEXT: Messages to understand flow +#define DEBUG_SOCKETS 0x04 // DBTEXT: Debug socket operations +#define DEBUG_INPUT 0x08 // DBTEXT: Debug client input +#define DEBUG_CLIENTS 0x10 // DBTEXT: Debug client arrival/depart +#define DEBUG_COMMANDS 0x20 // DBTEXT: Debug client commands +#define DEBUG_CHANNELS 0x40 // DBTEXT: Debug channel operations +#define DEBUG_STATUS 0x80 // DBTEXT: Debug status messages + +#define DEBUG_ALL 0xffffffff + +//int set_debug(char *arg); /* Returns 0 on success, -1 on failure */ + + +#endif /* _DEBUG_H_ */ diff --git a/utils/fnv.h b/utils/fnv.h new file mode 100644 index 0000000..34f35ca --- /dev/null +++ b/utils/fnv.h @@ -0,0 +1,225 @@ +/* + * fnv - Fowler/Noll/Vo- hash code + * + * @(#) $Revision: 5.4 $ + * @(#) $Id: fnv.h,v 5.4 2009/07/30 22:49:13 chongo Exp $ + * @(#) $Source: /usr/local/src/cmd/fnv/RCS/fnv.h,v $ + * + *** + * + * Fowler/Noll/Vo- hash + * + * The basis of this hash algorithm was taken from an idea sent + * as reviewer comments to the IEEE POSIX P1003.2 committee by: + * + * Phong Vo (http://www.research.att.com/info/kpv/) + * Glenn Fowler (http://www.research.att.com/~gsf/) + * + * In a subsequent ballot round: + * + * Landon Curt Noll (http://www.isthe.com/chongo/) + * + * improved on their algorithm. Some people tried this hash + * and found that it worked rather well. In an EMail message + * to Landon, they named it the ``Fowler/Noll/Vo'' or FNV hash. + * + * FNV hashes are designed to be fast while maintaining a low + * collision rate. The FNV speed allows one to quickly hash lots + * of data while maintaining a reasonable collision rate. See: + * + * http://www.isthe.com/chongo/tech/comp/fnv/index.html + * + * for more details as well as other forms of the FNV hash. + * + *** + * + * NOTE: The FNV-0 historic hash is not recommended. One should use + * the FNV-1 hash instead. + * + * To use the 32 bit FNV-0 historic hash, pass FNV0_32_INIT as the + * Fnv32_t hashval argument to fnv_32_buf() or fnv_32_str(). + * + * To use the 64 bit FNV-0 historic hash, pass FNV0_64_INIT as the + * Fnv64_t hashval argument to fnv_64_buf() or fnv_64_str(). + * + * To use the recommended 32 bit FNV-1 hash, pass FNV1_32_INIT as the + * Fnv32_t hashval argument to fnv_32_buf() or fnv_32_str(). + * + * To use the recommended 64 bit FNV-1 hash, pass FNV1_64_INIT as the + * Fnv64_t hashval argument to fnv_64_buf() or fnv_64_str(). + * + * To use the recommended 32 bit FNV-1a hash, pass FNV1_32A_INIT as the + * Fnv32_t hashval argument to fnv_32a_buf() or fnv_32a_str(). + * + * To use the recommended 64 bit FNV-1a hash, pass FNV1A_64_INIT as the + * Fnv64_t hashval argument to fnv_64a_buf() or fnv_64a_str(). + * + *** + * + * Please do not copyright this code. This code is in the public domain. + * + * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO + * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR + * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + * + * By: + * chongo /\oo/\ + * http://www.isthe.com/chongo/ + * + * Share and Enjoy! :-) + */ + +#if !defined(__FNV_H__) +#define __FNV_H__ + +#include + +#define FNV_VERSION "5.0.2" /* @(#) FNV Version */ + + +/* + * 32 bit FNV-0 hash type + */ +typedef u_int32_t Fnv32_t; + + +/* + * 32 bit FNV-0 zero initial basis + * + * This historic hash is not recommended. One should use + * the FNV-1 hash and initial basis instead. + */ +#define FNV0_32_INIT ((Fnv32_t)0) + + +/* + * 32 bit FNV-1 and FNV-1a non-zero initial basis + * + * The FNV-1 initial basis is the FNV-0 hash of the following 32 octets: + * + * chongo /\../\ + * + * NOTE: The \'s above are not back-slashing escape characters. + * They are literal ASCII backslash 0x5c characters. + * + * NOTE: The FNV-1a initial basis is the same value as FNV-1 by definition. + */ +#define FNV1_32_INIT ((Fnv32_t)0x811c9dc5) +#define FNV1_32A_INIT FNV1_32_INIT + + +/* + * 64 bit FNV-0 hash + */ +typedef u_int64_t Fnv64_t; + + +/* + * 64 bit FNV-0 zero initial basis + * + * This historic hash is not recommended. One should use + * the FNV-1 hash and initial basis instead. + */ +#define FNV0_64_INIT ((Fnv64_t)0) + + +/* + * 64 bit FNV-1 non-zero initial basis + * + * The FNV-1 initial basis is the FNV-0 hash of the following 32 octets: + * + * chongo /\../\ + * + * NOTE: The \'s above are not back-slashing escape characters. + * They are literal ASCII backslash 0x5c characters. + * + * NOTE: The FNV-1a initial basis is the same value as FNV-1 by definition. + */ +#define FNV1_64_INIT ((Fnv64_t)0xcbf29ce484222325ULL) +#define FNV1A_64_INIT FNV1_64_INIT + + +/* + * hash types + */ +enum fnv_type { + FNV_NONE = 0, /* invalid FNV hash type */ + FNV0_32 = 1, /* FNV-0 32 bit hash */ + FNV1_32 = 2, /* FNV-1 32 bit hash */ + FNV1a_32 = 3, /* FNV-1a 32 bit hash */ + FNV0_64 = 4, /* FNV-0 64 bit hash */ + FNV1_64 = 5, /* FNV-1 64 bit hash */ + FNV1a_64 = 6, /* FNV-1a 64 bit hash */ +}; + + +/* + * these test vectors are used as part o the FNV test suite + */ +struct test_vector { + void *buf; /* start of test vector buffer */ + int len; /* length of test vector */ +}; +struct fnv0_32_test_vector { + struct test_vector *test; /* test vector buffer to hash */ + Fnv32_t fnv0_32; /* expected FNV-0 32 bit hash value */ +}; +struct fnv1_32_test_vector { + struct test_vector *test; /* test vector buffer to hash */ + Fnv32_t fnv1_32; /* expected FNV-1 32 bit hash value */ +}; +struct fnv1a_32_test_vector { + struct test_vector *test; /* test vector buffer to hash */ + Fnv32_t fnv1a_32; /* expected FNV-1a 32 bit hash value */ +}; +struct fnv0_64_test_vector { + struct test_vector *test; /* test vector buffer to hash */ + Fnv64_t fnv0_64; /* expected FNV-0 64 bit hash value */ +}; +struct fnv1_64_test_vector { + struct test_vector *test; /* test vector buffer to hash */ + Fnv64_t fnv1_64; /* expected FNV-1 64 bit hash value */ +}; +struct fnv1a_64_test_vector { + struct test_vector *test; /* test vector buffer to hash */ + Fnv64_t fnv1a_64; /* expected FNV-1a 64 bit hash value */ +}; + + +/* + * external functions + */ +/* hash_32.c */ +extern Fnv32_t fnv_32_buf(void *buf, size_t len, Fnv32_t hashval); +extern Fnv32_t fnv_32_str(char *buf, Fnv32_t hashval); + +/* hash_32a.c */ +extern Fnv32_t fnv_32a_buf(void *buf, size_t len, Fnv32_t hashval); +extern Fnv32_t fnv_32a_str(char *buf, Fnv32_t hashval); + +/* hash_64.c */ +extern Fnv64_t fnv_64_buf(void *buf, size_t len, Fnv64_t hashval); +extern Fnv64_t fnv_64_str(char *buf, Fnv64_t hashval); + +/* hash_64a.c */ +extern Fnv64_t fnv_64a_buf(void *buf, size_t len, Fnv64_t hashval); +extern Fnv64_t fnv_64a_str(char *buf, Fnv64_t hashval); + +/* test_fnv.c */ +extern struct test_vector fnv_test_str[]; +extern struct fnv0_32_test_vector fnv0_32_vector[]; +extern struct fnv1_32_test_vector fnv1_32_vector[]; +extern struct fnv1a_32_test_vector fnv1a_32_vector[]; +extern struct fnv0_64_test_vector fnv0_64_vector[]; +extern struct fnv1_64_test_vector fnv1_64_vector[]; +extern struct fnv1a_64_test_vector fnv1a_64_vector[]; +extern void unknown_hash_type(char *prog, enum fnv_type type, int code); +extern void print_fnv32(Fnv32_t hval, Fnv32_t mask, int verbose, char *arg); +extern void print_fnv64(Fnv64_t hval, Fnv64_t mask, int verbose, char *arg); + + +#endif /* __FNV_H__ */ diff --git a/utils/hashutil.cc b/utils/hashutil.cc new file mode 100644 index 0000000..c427f8b --- /dev/null +++ b/utils/hashutil.cc @@ -0,0 +1,715 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// Pulled from lookup3.c by Bob Jenkins +#include "hashutil.h" + +namespace fawn { + /* + ------------------------------------------------------------------------------- + hashlittle() -- hash a variable-length key into a 32-bit value + k : the key (the unaligned variable-length array of bytes) + length : the length of the key, counting by bytes + initval : can be any 4-byte value + Returns a 32-bit value. Every bit of the key affects every bit of + the return value. Two keys differing by one or two bits will have + totally different hash values. + + The best hash table sizes are powers of 2. There is no need to do + mod a prime (mod is sooo slow!). If you need less than 32 bits, + use a bitmask. For example, if you need only 10 bits, do + h = (h & hashmask(10)); + In which case, the hash table should have hashsize(10) elements. + + If you are hashing n strings (uint8_t **)k, do it like this: + for (i=0, h=0; i 12) + { + a += k[0]; + b += k[1]; + c += k[2]; + mix(a,b,c); + length -= 12; + k += 3; + } + + /*----------------------------- handle the last (probably partial) block */ + /* + * "k[2]&0xffffff" actually reads beyond the end of the string, but + * then masks off the part it's not allowed to read. Because the + * string is aligned, the masked-off tail is in the same word as the + * rest of the string. Every machine with memory protection I've seen + * does it on word boundaries, so is OK with this. But VALGRIND will + * still catch it and complain. The masking trick does make the hash + * noticably faster for short strings (like English words). + */ +#ifndef VALGRIND + + switch(length) + { + case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; + case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break; + case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break; + case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break; + case 8 : b+=k[1]; a+=k[0]; break; + case 7 : b+=k[1]&0xffffff; a+=k[0]; break; + case 6 : b+=k[1]&0xffff; a+=k[0]; break; + case 5 : b+=k[1]&0xff; a+=k[0]; break; + case 4 : a+=k[0]; break; + case 3 : a+=k[0]&0xffffff; break; + case 2 : a+=k[0]&0xffff; break; + case 1 : a+=k[0]&0xff; break; + case 0 : return c; /* zero length strings require no mixing */ + } + +#else /* make valgrind happy */ + + const u_int8_t *k8; + k8 = (const u_int8_t *)k; + switch(length) + { + case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; + case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ + case 10: c+=((uint32_t)k8[9])<<8; /* fall through */ + case 9 : c+=k8[8]; /* fall through */ + case 8 : b+=k[1]; a+=k[0]; break; + case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ + case 6 : b+=((uint32_t)k8[5])<<8; /* fall through */ + case 5 : b+=k8[4]; /* fall through */ + case 4 : a+=k[0]; break; + case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ + case 2 : a+=((uint32_t)k8[1])<<8; /* fall through */ + case 1 : a+=k8[0]; break; + case 0 : return c; + } + +#endif /* !valgrind */ + + } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) { + const u_int16_t *k = (const u_int16_t *)buf; /* read 16-bit chunks */ + const u_int8_t *k8; + + /*--------------- all but last block: aligned reads and different mixing */ + while (length > 12) + { + a += k[0] + (((uint32_t)k[1])<<16); + b += k[2] + (((uint32_t)k[3])<<16); + c += k[4] + (((uint32_t)k[5])<<16); + mix(a,b,c); + length -= 12; + k += 6; + } + + /*----------------------------- handle the last (probably partial) block */ + k8 = (const u_int8_t *)k; + switch(length) + { + case 12: c+=k[4]+(((uint32_t)k[5])<<16); + b+=k[2]+(((uint32_t)k[3])<<16); + a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ + case 10: c+=k[4]; + b+=k[2]+(((uint32_t)k[3])<<16); + a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 9 : c+=k8[8]; /* fall through */ + case 8 : b+=k[2]+(((uint32_t)k[3])<<16); + a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ + case 6 : b+=k[2]; + a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 5 : b+=k8[4]; /* fall through */ + case 4 : a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ + case 2 : a+=k[0]; + break; + case 1 : a+=k8[0]; + break; + case 0 : return c; /* zero length requires no mixing */ + } + + } else { /* need to read the key one byte at a time */ + const u_int8_t *k = (const u_int8_t *)buf; + + /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ + while (length > 12) + { + a += k[0]; + a += ((uint32_t)k[1])<<8; + a += ((uint32_t)k[2])<<16; + a += ((uint32_t)k[3])<<24; + b += k[4]; + b += ((uint32_t)k[5])<<8; + b += ((uint32_t)k[6])<<16; + b += ((uint32_t)k[7])<<24; + c += k[8]; + c += ((uint32_t)k[9])<<8; + c += ((uint32_t)k[10])<<16; + c += ((uint32_t)k[11])<<24; + mix(a,b,c); + length -= 12; + k += 12; + } + + /*-------------------------------- last block: affect all 32 bits of (c) */ + switch(length) /* all the case statements fall through */ + { + case 12: c+=((uint32_t)k[11])<<24; + case 11: c+=((uint32_t)k[10])<<16; + case 10: c+=((uint32_t)k[9])<<8; + case 9 : c+=k[8]; + case 8 : b+=((uint32_t)k[7])<<24; + case 7 : b+=((uint32_t)k[6])<<16; + case 6 : b+=((uint32_t)k[5])<<8; + case 5 : b+=k[4]; + case 4 : a+=((uint32_t)k[3])<<24; + case 3 : a+=((uint32_t)k[2])<<16; + case 2 : a+=((uint32_t)k[1])<<8; + case 1 : a+=k[0]; + break; + case 0 : return c; + } + } + + final(a,b,c); + return c; + } + + + /* + * hashlittle2: return 2 32-bit hash values + * + * This is identical to hashlittle(), except it returns two 32-bit hash + * values instead of just one. This is good enough for hash table + * lookup with 2^^64 buckets, or if you want a second hash if you're not + * happy with the first, or if you want a probably-unique 64-bit ID for + * the key. *pc is better mixed than *pb, so use *pc first. If you want + * a 64-bit value do something like "*pc + (((uint64_t)*pb)<<32)". + */ + void HashUtil::BobHash(const void *buf, size_t length, uint32_t *idx1, uint32_t *idx2) + { + uint32_t a,b,c; /* internal state */ + union { const void *ptr; size_t i; } u; /* needed for Mac Powerbook G4 */ + + /* Set up the internal state */ + a = b = c = 0xdeadbeef + ((uint32_t)length) + *idx1; + c += *idx2; + + u.ptr = buf; + if (HASH_LITTLE_ENDIAN && ((u.i & 0x3) == 0)) { + const uint32_t *k = (const uint32_t *)buf; /* read 32-bit chunks */ + const uint8_t *k8; + + /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ + while (length > 12) + { + a += k[0]; + b += k[1]; + c += k[2]; + mix(a,b,c); + length -= 12; + k += 3; + } + + /*----------------------------- handle the last (probably partial) block */ + /* + * "k[2]&0xffffff" actually reads beyond the end of the string, but + * then masks off the part it's not allowed to read. Because the + * string is aligned, the masked-off tail is in the same word as the + * rest of the string. Every machine with memory protection I've seen + * does it on word boundaries, so is OK with this. But VALGRIND will + * still catch it and complain. The masking trick does make the hash + * noticably faster for short strings (like English words). + */ +#ifndef VALGRIND + + switch(length) + { + case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; + case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break; + case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break; + case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break; + case 8 : b+=k[1]; a+=k[0]; break; + case 7 : b+=k[1]&0xffffff; a+=k[0]; break; + case 6 : b+=k[1]&0xffff; a+=k[0]; break; + case 5 : b+=k[1]&0xff; a+=k[0]; break; + case 4 : a+=k[0]; break; + case 3 : a+=k[0]&0xffffff; break; + case 2 : a+=k[0]&0xffff; break; + case 1 : a+=k[0]&0xff; break; + case 0 : *idx1=c; *idx2=b; return; /* zero length strings require no mixing */ + } + +#else /* make valgrind happy */ + + k8 = (const uint8_t *)k; + switch(length) + { + case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; + case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ + case 10: c+=((uint32_t)k8[9])<<8; /* fall through */ + case 9 : c+=k8[8]; /* fall through */ + case 8 : b+=k[1]; a+=k[0]; break; + case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ + case 6 : b+=((uint32_t)k8[5])<<8; /* fall through */ + case 5 : b+=k8[4]; /* fall through */ + case 4 : a+=k[0]; break; + case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ + case 2 : a+=((uint32_t)k8[1])<<8; /* fall through */ + case 1 : a+=k8[0]; break; + case 0 : *idx1=c; *idx2=b; return; /* zero length strings require no mixing */ + } + +#endif /* !valgrind */ + + } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) { + const uint16_t *k = (const uint16_t *)buf; /* read 16-bit chunks */ + const uint8_t *k8; + + /*--------------- all but last block: aligned reads and different mixing */ + while (length > 12) + { + a += k[0] + (((uint32_t)k[1])<<16); + b += k[2] + (((uint32_t)k[3])<<16); + c += k[4] + (((uint32_t)k[5])<<16); + mix(a,b,c); + length -= 12; + k += 6; + } + + /*----------------------------- handle the last (probably partial) block */ + k8 = (const uint8_t *)k; + switch(length) + { + case 12: c+=k[4]+(((uint32_t)k[5])<<16); + b+=k[2]+(((uint32_t)k[3])<<16); + a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ + case 10: c+=k[4]; + b+=k[2]+(((uint32_t)k[3])<<16); + a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 9 : c+=k8[8]; /* fall through */ + case 8 : b+=k[2]+(((uint32_t)k[3])<<16); + a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ + case 6 : b+=k[2]; + a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 5 : b+=k8[4]; /* fall through */ + case 4 : a+=k[0]+(((uint32_t)k[1])<<16); + break; + case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ + case 2 : a+=k[0]; + break; + case 1 : a+=k8[0]; + break; + case 0 : *idx1=c; *idx2=b; return; /* zero length strings require no mixing */ + } + + } else { /* need to read the key one byte at a time */ + const uint8_t *k = (const uint8_t *)buf; + + /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ + while (length > 12) + { + a += k[0]; + a += ((uint32_t)k[1])<<8; + a += ((uint32_t)k[2])<<16; + a += ((uint32_t)k[3])<<24; + b += k[4]; + b += ((uint32_t)k[5])<<8; + b += ((uint32_t)k[6])<<16; + b += ((uint32_t)k[7])<<24; + c += k[8]; + c += ((uint32_t)k[9])<<8; + c += ((uint32_t)k[10])<<16; + c += ((uint32_t)k[11])<<24; + mix(a,b,c); + length -= 12; + k += 12; + } + + /*-------------------------------- last block: affect all 32 bits of (c) */ + switch(length) /* all the case statements fall through */ + { + case 12: c+=((uint32_t)k[11])<<24; + case 11: c+=((uint32_t)k[10])<<16; + case 10: c+=((uint32_t)k[9])<<8; + case 9 : c+=k[8]; + case 8 : b+=((uint32_t)k[7])<<24; + case 7 : b+=((uint32_t)k[6])<<16; + case 6 : b+=((uint32_t)k[5])<<8; + case 5 : b+=k[4]; + case 4 : a+=((uint32_t)k[3])<<24; + case 3 : a+=((uint32_t)k[2])<<16; + case 2 : a+=((uint32_t)k[1])<<8; + case 1 : a+=k[0]; + break; + case 0 : *idx1=c; *idx2=b; return; /* zero length strings require no mixing */ + } + } + + final(a,b,c); + *idx1=c; *idx2=b; + } + + void HashUtil::BobHash(const string &s, uint32_t *idx1, uint32_t *idx2) + { + return BobHash(s.data(), s.length(), idx1, idx2); + } + + /* + * Integer Hashing + * http://burtleburtle.net/bob/hash/integer.html + */ + + /* + * 4-byte integer hash, full avalanche with 6 shifts and magic constatnts + */ + uint32_t HashUtil::hashint_full_avalanche_1( uint32_t a) + { + a = (a+0x7ed55d16) + (a<<12); + a = (a^0xc761c23c) ^ (a>>19); + a = (a+0x165667b1) + (a<<5); + a = (a+0xd3a2646c) ^ (a<<9); + a = (a+0xfd7046c5) + (a<<3); + a = (a^0xb55a4f09) ^ (a>>16); + return a; + } + + /* + * 4-byte integer hash, full avalanche with 7 shifts and NO magic constatnts + */ + uint32_t HashUtil::hashint_full_avalanche_2( uint32_t a) + { + a -= (a<<6); + a ^= (a>>17); + a -= (a<<9); + a ^= (a<<4); + a -= (a<<3); + a ^= (a<<10); + a ^= (a>>15); + return a; + } + + uint32_t HashUtil::hashint_half_avalanche( uint32_t a) + { + a = (a+0x479ab41d) + (a<<8); + a = (a^0xe4aa10ce) ^ (a>>5); + a = (a+0x9942f0a6) - (a<<14); + a = (a^0x5aedd67d) ^ (a>>3); + a = (a+0x17bea992) + (a<<7); + return a; + } + + /* + * hash_32 - 32 bit Fowler/Noll/Vo hash code + * + * @(#) $Revision: 5.1 $ + * @(#) $Id: hash_32.c,v 5.1 2009/06/30 09:13:32 chongo Exp $ + * @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_32.c,v $ + * + *** + * + * Fowler/Noll/Vo hash + * + * The basis of this hash algorithm was taken from an idea sent + * as reviewer comments to the IEEE POSIX P1003.2 committee by: + * + * Phong Vo (http://www.research.att.com/info/kpv/) + * Glenn Fowler (http://www.research.att.com/~gsf/) + * + * In a subsequent ballot round: + * + * Landon Curt Noll (http://www.isthe.com/chongo/) + * + * improved on their algorithm. Some people tried this hash + * and found that it worked rather well. In an EMail message + * to Landon, they named it the ``Fowler/Noll/Vo'' or FNV hash. + * + * FNV hashes are designed to be fast while maintaining a low + * collision rate. The FNV speed allows one to quickly hash lots + * of data while maintaining a reasonable collision rate. See: + * + * http://www.isthe.com/chongo/tech/comp/fnv/index.html + * + * for more details as well as other forms of the FNV hash. + *** + * + * NOTE: The FNV-0 historic hash is not recommended. One should use + * the FNV-1 hash instead. + * + * To use the 32 bit FNV-0 historic hash, pass FNV0_32_INIT as the + * uint32_t hashval argument to fnv_32_buf() or fnv_32_str(). + * + * To use the recommended 32 bit FNV-1 hash, pass FNV1_32_INIT as the + * uint32_t hashval argument to fnv_32_buf() or fnv_32_str(). + * + *** + * + * Please do not copyright this code. This code is in the public domain. + * + * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO + * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR + * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + * + * By: + * chongo /\oo/\ + * http://www.isthe.com/chongo/ + * + * Share and Enjoy! :-) + */ + + /* + * 32 bit magic FNV-0 and FNV-1 prime + */ +#define FNV_32_PRIME ((uint32_t)0x01000193) + + + /* + * fnv_32_buf - perform a 32 bit Fowler/Noll/Vo hash on a buffer + * + * input: + * buf - start of buffer to hash + * len - length of buffer in octets + * hval - previous hash value or 0 if first call + * + * returns: + * 32 bit hash as a static hash type + * + * NOTE: To use the 32 bit FNV-0 historic hash, use FNV0_32_INIT as the hval + * argument on the first call to either fnv_32_buf() or fnv_32_str(). + * + * NOTE: To use the recommended 32 bit FNV-1 hash, use FNV1_32_INIT as the hval + * argument on the first call to either fnv_32_buf() or fnv_32_str(). + * + * vrv: hardcoded the use of FNV1_32_INIT + */ + uint32_t HashUtil::FNVHash(const void *buf, size_t len) + { + uint32_t hval = FNV1_32_INIT; + unsigned char *bp = (unsigned char *)buf; /* start of buffer */ + unsigned char *be = bp + len; /* beyond end of buffer */ + + /* + * FNV-1 hash each octet in the buffer + */ + while (bp < be) { + + /* multiply by the 32 bit FNV magic prime mod 2^32 */ +#if defined(NO_FNV_GCC_OPTIMIZATION) + hval *= FNV_32_PRIME; +#else + hval += (hval<<1) + (hval<<4) + (hval<<7) + (hval<<8) + (hval<<24); +#endif + + /* xor the bottom with the current octet */ + hval ^= (uint32_t)*bp++; + } + + /* return our new hash value */ + return hval; + } + + uint32_t HashUtil::FNVHash(const string &s) + { + return FNVHash(s.data(), s.length()); + } + + + //----------------------------------------------------------------------------- + // MurmurHash2, by Austin Appleby + // Note - This code makes a few assumptions about how your machine behaves - + // 1. We can read a 4-byte value from any address without crashing + // 2. sizeof(int) == 4 + // And it has a few limitations - + // 1. It will not work incrementally. + // 2. It will not produce the same results on little-endian and big-endian + // machines. + // All code is released to the public domain. For business purposes, + // Murmurhash is under the MIT license. + + + uint32_t HashUtil::MurmurHash(const void* buf, size_t len, uint32_t seed) + { + // 'm' and 'r' are mixing constants generated offline. + // They're not really 'magic', they just happen to work well. + + const unsigned int m = 0x5bd1e995; + const int r = 24; + + // Initialize the hash to a 'random' value + uint32_t h = seed ^ len; + + // Mix 4 bytes at a time into the hash + const unsigned char * data = (const unsigned char *)buf; + + while(len >= 4) { + unsigned int k = *(unsigned int *)data; + + k *= m; + k ^= k >> r; + k *= m; + + h *= m; + h ^= k; + + data += 4; + len -= 4; + } + + // Handle the last few bytes of the input array + switch(len) { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; + h *= m; + }; + + // Do a few final mixes of the hash to ensure the last few + // bytes are well-incorporated. + h ^= h >> 13; + h *= m; + h ^= h >> 15; + return h; + } + + uint32_t HashUtil::MurmurHash(const string &s, uint32_t seed) + { + return MurmurHash(s.data(), s.length(), seed); + } + + + // SuperFastHash aka Hsieh Hash, License: GPL 2.0 + uint32_t HashUtil::SuperFastHash(const void *buf, size_t len) + { + const char* data = (const char*) buf; + uint32_t hash = len, tmp; + int rem; + + if (len <= 0 || data == NULL) return 0; + + rem = len & 3; + len >>= 2; + + /* Main loop */ + for (;len > 0; len--) { + hash += get16bits (data); + tmp = (get16bits (data+2) << 11) ^ hash; + hash = (hash << 16) ^ tmp; + data += 2*sizeof (uint16_t); + hash += hash >> 11; + } + + /* Handle end cases */ + switch (rem) { + case 3: hash += get16bits (data); + hash ^= hash << 16; + hash ^= data[sizeof (uint16_t)] << 18; + hash += hash >> 11; + break; + case 2: hash += get16bits (data); + hash ^= hash << 11; + hash += hash >> 17; + break; + case 1: hash += *data; + hash ^= hash << 10; + hash += hash >> 1; + } + + /* Force "avalanching" of final 127 bits */ + hash ^= hash << 3; + hash += hash >> 5; + hash ^= hash << 4; + hash += hash >> 17; + hash ^= hash << 25; + hash += hash >> 6; + + return hash; + } + + uint32_t HashUtil::SuperFastHash(const string &s) + { + return SuperFastHash(s.data(), s.length()); + } + + uint32_t HashUtil::NullHash(const void* buf, size_t length, uint32_t shiftbytes) + { + // Ensure that enough bits exist in buffer + if (length - shiftbytes < sizeof(uint32_t)) { + return 0; + } + char* data = (char*) buf; + return ((data[(length-shiftbytes-4)] << 24) + + (data[(length-shiftbytes-3)] << 16) + + (data[(length-shiftbytes-2)] << 8) + + (data[(length-shiftbytes-1)])); + } + + std::string HashUtil::MD5Hash(const char* inbuf, size_t in_length) + { + EVP_MD_CTX mdctx; + unsigned char md_value[EVP_MAX_MD_SIZE]; + unsigned int md_len; + + EVP_DigestInit(&mdctx, EVP_md5()); + EVP_DigestUpdate(&mdctx, (const void*) inbuf, in_length); + EVP_DigestFinal_ex(&mdctx, md_value, &md_len); + EVP_MD_CTX_cleanup(&mdctx); + + return string((char*)md_value, (size_t)md_len); + } + + + std::string HashUtil::SHA1Hash(const char* inbuf, size_t in_length) + { + EVP_MD_CTX mdctx; + string ret; + unsigned char md_value[EVP_MAX_MD_SIZE]; + unsigned int md_len; + + EVP_DigestInit(&mdctx, EVP_sha1()); + EVP_DigestUpdate(&mdctx, (const void*) inbuf, in_length); + EVP_DigestFinal_ex(&mdctx, md_value, &md_len); + EVP_MD_CTX_cleanup(&mdctx); + + return string((char*)md_value, (size_t)md_len); + } +} // namespace fawn diff --git a/utils/hashutil.h b/utils/hashutil.h new file mode 100644 index 0000000..c214ea0 --- /dev/null +++ b/utils/hashutil.h @@ -0,0 +1,97 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _HASHUTIL_H_ +#define _HASHUTIL_H_ + +#include +#include +#include +#include +#include +#include "fnv.h" + +using namespace std; + +#define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k)))) +#define mix(a,b,c) \ + { \ + a -= c; a ^= rot(c, 4); c += b; \ + b -= a; b ^= rot(a, 6); a += c; \ + c -= b; c ^= rot(b, 8); b += a; \ + a -= c; a ^= rot(c,16); c += b; \ + b -= a; b ^= rot(a,19); a += c; \ + c -= b; c ^= rot(b, 4); b += a; \ + } + +#define final(a,b,c) \ + { \ + c ^= b; c -= rot(b,14); \ + a ^= c; a -= rot(c,11); \ + b ^= a; b -= rot(a,25); \ + c ^= b; c -= rot(b,16); \ + a ^= c; a -= rot(c,4); \ + b ^= a; b -= rot(a,14); \ + c ^= b; c -= rot(b,24); \ + } + // Assuming little endian +#define HASH_LITTLE_ENDIAN 1 + +#define get16bits(d) (*((const uint16_t *) (d))) + +namespace fawn { + class HashUtil { + public: + // Bob Jenkins Hash + static uint32_t BobHash(const void *buf, size_t length, uint32_t seed = 0); + static uint32_t BobHash(const string &s, uint32_t seed = 0); + + // Bob Jenkins Hash that returns two indices in one call + // Useful for Cuckoo hashing, power of two choices, etc. + // Use idx1 before idx2, when possible. idx1 and idx2 should be initialized to seeds. + static void BobHash(const void *buf, size_t length, uint32_t *idx1, uint32_t *idx2); + static void BobHash(const string &s, uint32_t *idx1, uint32_t *idx2); + + // MurmurHash2 + static uint32_t MurmurHash(const void *buf, size_t length, uint32_t seed = 0); + static uint32_t MurmurHash(const string &s, uint32_t seed = 0); + + // FNV Hash + static uint32_t FNVHash(const void *buf, size_t length); + static uint32_t FNVHash(const string &s); + + // SuperFastHash + static uint32_t SuperFastHash(const void *buf, size_t len); + static uint32_t SuperFastHash(const string &s); + + // Integer hashes (from Bob Jenkins) + static uint32_t hashint_full_avalanche_1( uint32_t a); + static uint32_t hashint_full_avalanche_2( uint32_t a); + static uint32_t hashint_half_avalanche( uint32_t a); + + // Null hash (shift and mask) + static uint32_t NullHash(const void* buf, size_t length, uint32_t shiftbytes); + + // Wrappers for MD5 and SHA1 hashing using EVP + static std::string MD5Hash(const char* inbuf, size_t in_length); + static std::string SHA1Hash(const char* inbuf, size_t in_length); + + static uint32_t FindNextHashSize(uint32_t number) + { + // Gets the next highest power of 2 larger than number + number--; + number = (number >> 1) | number; + number = (number >> 2) | number; + number = (number >> 4) | number; + number = (number >> 8) | number; + number = (number >> 16) | number; + number++; + return number; + } + + private: + HashUtil(); + }; +} + +#endif // #ifndef _HASHUTIL_H_ + + diff --git a/utils/print.cc b/utils/print.cc new file mode 100644 index 0000000..22dfd66 --- /dev/null +++ b/utils/print.cc @@ -0,0 +1,231 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#include +#include +#include "print.h" + +void bytes_into_hex_string(const u_char *data, u_int len, string& dststr) +{ + static const char hexes[] = "0123456789ABCDEF"; + dststr.reserve(dststr.size() + len*2); + + for (u_int i = 0; i < len; i++) { + dststr.push_back(hexes[data[i] >> 4]); + dststr.push_back(hexes[data[i] & 0xf]); + } +} + +/* May incur a copy; don't use on high-performance path unless you know + * str is refcounted */ +string bytes_to_hex(const u_char* data, u_int len) +{ + string r; + bytes_into_hex_string(data, len, r); + return r; +} + +string bytes_to_hex(const string& s) +{ + return bytes_to_hex((const u_char *)s.data(), s.size()); +} + + +int get_digit_value(char digit) +{ + if (isdigit(digit)) + { + return digit - '0'; + } + else if (digit >= 'A' && digit <= 'F') + { + return digit - 'A' + 10; + } + else if (digit >= 'a' && digit <= 'f') + { + return digit - 'a' + 10; + } + else // illegal digit + { + return -1; + } +} + +// assumes 2 character string with legal hex digits +string* hex_string_to_bytes(const u_char* hex_num, u_int len) +{ + string* p_ret = new string(); + for (u_int i = 0; i < len/2; i++) { + char high = hex_num[i]; + char low = hex_num[i+1]; + int low_val = get_digit_value(low); //convert low to number from 0 to 15 + int high_val = get_digit_value(high); //convert high to number from 0 to 15 + + if ((low_val < 0) || (high_val < 0)) { + // Narf! + delete p_ret; + return NULL; + } + + char ch = low_val + 16 * high_val; + p_ret->append(1, ch); + } + return p_ret; +} + + +/* + * print data in rows of 16 bytes: offset hex ascii + * + * 00000 47 45 54 20 2f 20 48 54 54 50 2f 31 2e 31 0d 0a GET / HTTP/1.1.. + */ +void print_hex_ascii_line(const u_char *payload, + u_int len, + u_int offset) +{ + /* offset */ + fprintf(stderr, "%05d ", offset); + + /* hex */ + const u_char *ch = payload; + for (u_int i = 0; i < len; i++) { + fprintf(stderr, "%02x ", *ch); + ch++; + /* print extra space after 8th byte for visual aid */ + if (i == 7) + fprintf(stderr, " "); + } + /* print space to handle line less than 8 bytes */ + if (len < 8) + fprintf(stderr, " "); + + /* fill hex gap with spaces if not full line */ + if (len < 16) { + int gap = 16 - len; + for (int i = 0; i < gap; i++) { + fprintf(stderr, " "); + } + } + fprintf(stderr, " "); + + /* ascii (if printable) */ + ch = payload; + for (u_int i = 0; i < len; i++) { + if (isprint(*ch)) + fprintf(stderr, "%c", *ch); + else + fprintf(stderr, "."); + ch++; + } + + fprintf(stderr, "\n"); + + return; +} + +void print_payload(const string &str, int indent) { + print_payload((const u_char*)str.data(), str.size(), indent); +} +/* + * print packet payload data (avoid printing binary data) + */ +void print_payload(const u_char *payload, u_int len, int indent) +{ + const u_int line_width = 16; /* number of bytes per line */ + u_int len_rem = len; + u_int offset = 0; + + while (len_rem > line_width) { + int line_len = line_width % len_rem; + fprintf(stderr, "%*s", indent, ""); + print_hex_ascii_line(payload + offset, line_len, offset); + len_rem -= line_len; + offset += line_width; + } + + /* Might have left a partial line left. */ + if (len_rem > 0) { + fprintf(stderr, "%*s", indent, ""); + print_hex_ascii_line(payload + offset, len_rem, offset); + } + + return; +} + + + +void tokenize(const string& str, + vector& tokens, + const string& delimiters) +{ + // Skip delimiters at beginning. + string::size_type lastPos = str.find_first_not_of(delimiters, 0); + // Find first "non-delimiter". + string::size_type pos = str.find_first_of(delimiters, lastPos); + + while (string::npos != pos || string::npos != lastPos) + { + // Found a token, add it to the vector. + tokens.push_back(str.substr(lastPos, pos - lastPos)); + // Skip delimiters. Note the "not_of" + lastPos = str.find_first_not_of(delimiters, pos); + // Find next "non-delimiter" + pos = str.find_first_of(delimiters, lastPos); + } +} + +void int_to_byte(const uint32_t i, char* p_byte_value) +{ + uint32_t int_value = i; + for (int x = 0; x < 4; x++) { + p_byte_value[x] = int_value & 0x000000FF; + int_value = int_value >> 8; + cout << "b[" << x << "] = " << hex << uppercase << setw(2) << setfill('0') <<(int)p_byte_value[x] << endl; + } +} + + +void int_to_bit (const uint32_t i, int w ) +{ + uint32_t z; + string b=""; + for (z = 1 << (w-1); z > 0; z >>= 1) + { + b = (((i & z) == z) ? "1" : "0") + b; + } + cout << "bit[0.."<< w-1 << "]=" << b << endl; +} + +int fill_file_with_zeros(int fd, size_t nbytes) +{ + static const size_t BLOCKSIZE = 8192; + char zeros[BLOCKSIZE]; + memset(zeros, 0, BLOCKSIZE); + while (nbytes > 0) { + size_t bytes_to_write = min(nbytes, BLOCKSIZE); + ssize_t ret = write(fd, zeros, bytes_to_write); + if (ret < 0) { + perror("error in fill_file_with_zeros write"); + return -1; + } + nbytes -= bytes_to_write; + } + return 0; +} + +string getID(string ip, const int32_t port) +{ + char sport[7]; + sprintf(sport, "%d", port); + return ip.append(":").append(sport, strlen(sport)); +} + +string getIP(string id) +{ + return id.substr(0, id.find(":")); +} + +int getPort(string id) +{ + string p = id.substr(id.find(":")+1, id.size()); + return strtol(p.c_str(), NULL, 10); +} + diff --git a/utils/print.h b/utils/print.h new file mode 100644 index 0000000..2fd328e --- /dev/null +++ b/utils/print.h @@ -0,0 +1,37 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _PRINT_H_ +#define _PRINT_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +void bytes_into_hex_string(const u_char *data, u_int len, string &dststr); +string bytes_to_hex(const u_char* data, u_int len); +string bytes_to_hex(const string& s); +int get_digit_value(char digit); +string* hex_string_to_bytes(const u_char* hex_num, u_int len); +void print_hex_ascii_line(const u_char *payload, u_int len, u_int offset); +void print_payload(const u_char *payload, u_int len, int indent=0); +void print_payload(const string &str, size_t indent=0); +void tokenize(const string& str, vector& tokens, const string& delimiters); + +void int_to_byte(const uint32_t i, char* p_byte_value); +void int_to_bit (const uint32_t x, int w ); +int fill_file_with_zeros(int fd, size_t nbytes); + + +string getID(string ip, const int32_t port); +string getIP(string id); +int getPort(string id); + +#endif //_PRINT_H_ diff --git a/utils/timing.c b/utils/timing.c new file mode 100644 index 0000000..eca3a59 --- /dev/null +++ b/utils/timing.c @@ -0,0 +1,21 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* + * Some benchmarking utils + */ + +#include "timing.h" + +double timeval_diff(const struct timeval * const start, const struct timeval * const end) +{ + /* Calculate the second difference*/ + double r = end->tv_sec - start->tv_sec; + + /* Calculate the microsecond difference */ + if (end->tv_usec > start->tv_usec) + r += (end->tv_usec - start->tv_usec)/1000000.0; + else if (end->tv_usec < start->tv_usec) + r -= (start->tv_usec - end->tv_usec)/1000000.0; + + return r; +} + diff --git a/utils/timing.h b/utils/timing.h new file mode 100644 index 0000000..b302c6f --- /dev/null +++ b/utils/timing.h @@ -0,0 +1,43 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ifndef _TIMING_H_ +#define _TIMING_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + double timeval_diff(const struct timeval * const start, const struct timeval * const end); + + +#ifdef __cplusplus +} /* extern C */ +#endif + + +/* You'll have to change the HZ constant in the printf below... */ +#include +#include +#include + +#if defined(__i386__) + +static __inline__ unsigned long long rdtsc(void) +{ + unsigned long long int x; + __asm__ volatile (".byte 0x0f, 0x31" : "=A" (x)); + return x; +} +#elif defined(__x86_64__) + +static __inline__ unsigned long long rdtsc(void) +{ + unsigned hi, lo; + __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); + return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 ); +} +#endif + +#endif /* _TIMING_H_ */