Permalink
Browse files

Refactored AFF4Image writer and added a thread pool. (#55)

* Refactored AFF4Image implementation.
* Implemented multithreaded compression.
* Made tests pass.
  • Loading branch information...
scudette committed Jan 8, 2018
1 parent bf6bca7 commit 077964ab60b3025733cdc61f9425aae5641072ac
Showing with 657 additions and 314 deletions.
  1. +37 −5 README.linux
  2. +26 −0 configure.ac
  3. +17 −10 src/Makefile.am
  4. +5 −0 src/aff4_base.h
  5. +345 −252 src/aff4_image.cc
  6. +23 −16 src/aff4_image.h
  7. +6 −0 src/aff4_imager_utils.cc
  8. +5 −1 src/aff4_imager_utils.h
  9. +10 −4 src/aff4_io.h
  10. +3 −3 src/aff4imager.cc
  11. +15 −0 src/compatibility.cc
  12. +8 −7 src/data_store.cc
  13. +11 −1 src/data_store.h
  14. +8 −0 src/libaff4.cc
  15. +125 −0 src/threadpool.h
  16. +11 −6 src/zip.cc
  17. +1 −8 tests/Makefile.am
  18. +1 −1 tests/aff4_image_test.cc
@@ -1,15 +1,46 @@
Building statically for linux.
==============================
Building staticly for linux.
============================

The following explains how to build statically on Ubuntu 17.10.
The following explains how to build statically on Ubuntu
17.10. Totally static builds are no longer supported on Linux because
pthread does not work. So we need to make sure that everything is
compiled statically, but we will still have a dependency on glibc and
pthread.

apt-get install libtool-bin libboost-dev
A complete discussion of the issue cam be found here
http://insanecoding.blogspot.com/2012/07/creating-portable-linux-binaries.html

At the end we should have a binary which is mostly static:
⟫ ldd ./aff4imager
linux-vdso.so.1 => (0x00007ffef1bee000)
libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f6b64840000)
libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f6b64460000)
/lib64/ld-linux-x86-64.so.2 (0x00007f6b64ec7000)


# It is best to maintain a special prefix with static libraries so
# they do not pollute the system.
# they do not pollute the system. Since static libraries are not
# shipped any more, we need to build all dependencies from
# source. This only needs to be done once as we keep linking this
# directory in future.

export PREFIX=/home/scudette/build/static/
export CXXFLAGS="-I$PREFIX/include"
export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig/"

apt-get install libtool-bin libboost-dev

apt-get source uuid-dev
cd util-linux-2.30.1
./configure --prefix=$PREFIX --enable-static LDFLAGS="-L$PREFIX/lib -static -static-libstdc++ -static-libgcc" --enable-libuuid --disable-all-programs --disable-python --disable-bash-completion
make install -j4
cd..

apt-get source zlib1g
cd zlib-1.2.11.dfsg/
./configure --prefix=$PREFIX --static
make install
cd ..

apt-get source libraptor2-dev
cd raptor2-2.0.13/
@@ -61,6 +92,7 @@ cd aff4
make -j4 install
cd ..

# To build linpmem
git clone https://github.com/google/rekall.git
cd rekall/tools/pmem
./bootstrap
@@ -88,6 +88,32 @@ fi
# Check for windows.
MINGW_AC_WIN32_NATIVE_HOST
URIPARSER_CHECK
AC_CANONICAL_HOST

build_linux=no
build_windows=no
build_mac=no

# Detect the target system
case "${host_os}" in
linux*)
build_linux=yes
;;
cygwin*|mingw*)
build_windows=yes
;;
darwin*)
build_mac=yes
;;
*)
AC_MSG_ERROR(["OS $host_os is not supported"])
;;
esac

# Pass the conditionals to automake
AM_CONDITIONAL([LINUX], [test "$build_linux" = "yes"])
AM_CONDITIONAL([WINDOWS], [test "$build_windows" = "yes"])
AM_CONDITIONAL([OSX], [test "$build_mac" = "yes"])

# Set Makefile / config.h values.
AM_CONDITIONAL([HAVE_LIBYAML_CPP], [test "x$WITH_YAML" = "xyes"])
@@ -10,19 +10,20 @@ AM_CPPFLAGS = -I$(top_srcdir)/src

# On windows we need to specify this to avoid a bug in gcc
# (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=52991)
if WIN32_NATIVE_HOST
if WINDOWS
WINCXXFLAGS = -mno-ms-bitfields
else
UNIXLDFLAGS = ${UUID_CFLAGS}
endif


# Default flags for targets without specific overrides.
AM_CXXFLAGS = $(WINCXXFLAGS)

# Support static builds.
if STATIC_BUILD
STATIC_LIBLDFLAGS = -static -static-libstdc++ -static-libgcc
STATIC_LINKERLDFLAGS = -all-static
STATIC_LINKERLDFLAGS = -static -static-libstdc++ -static-libgcc
endif

# The -no-undefined flag allows libtool to generate shared libraries for
@@ -75,17 +76,23 @@ libaff4_la_SOURCES = \
libaff4-c.cc \
tclap_parsers.cc


libaff4_la_LDFLAGS = $(STATIC_LIBLDFLAGS)
libaff4_la_INCLUDES = ${RAPTOR2_CFLAGS} ${UUID_CFLAGS} ${TCLAP_CFLAGS} ${YAML_CPP_CFLAGS} ${ZLIB_CFLAGS} ${UUID_CFLAGS} ${GLOG_CFLAGS}
if WIN32_NATIVE_HOST
libaff4_la_LIBADD = -lrpcrt4 -lshlwapi ${LIBS} ${YAML_CPP_LIBS} ${ZLIB_LIBS} ${RAPTOR2_LIBS} ${TCLAP_LIBS} ${GLOG_LIBS}
libaff4_la_INCLUDES = ${RAPTOR2_CFLAGS} ${UUID_CFLAGS} ${TCLAP_CFLAGS} ${YAML_CPP_CFLAGS} ${ZLIB_CFLAGS} ${UUID_CFLAGS}

if WINDOWS
libaff4_la_LIBADD = -lrpcrt4 -lshlwapi ${LIBS} ${YAML_CPP_LIBS} ${ZLIB_LIBS} ${RAPTOR2_LIBS} ${TCLAP_LIBS}
else
libaff4_la_LIBADD = ${LIBS} ${YAML_CPP_LIBS} ${ZLIB_LIBS} ${RAPTOR2_LIBS} ${TCLAP_LIBS} ${UUID_LIBS} ${GLOG_LIBS}
libaff4_la_LIBADD = ${LIBS} ${YAML_CPP_LIBS} ${ZLIB_LIBS} ${RAPTOR2_LIBS} ${TCLAP_LIBS} ${UUID_LIBS}
endif

bin_PROGRAMS = aff4imager
aff4imager_LDADD = libaff4.la ${LIBS} ${YAML_CPP_LIBS} ${ZLIB_LIBS} ${RAPTOR2_LIBS} ${TCLAP_LIBS} ${GLOG_LIBS} ${UUID_LIBS}
aff4imager_INCLUDES = ${RAPTOR2_CFLAGS} ${UUID_CFLAGS} ${TCLAP_CFLAGS} ${YAML_CPP_CFLAGS} ${ZLIB_CFLAGS} ${GLOG_CFLAGS}
aff4imager_LDADD = libaff4.la ${LIBS} ${YAML_CPP_LIBS} ${ZLIB_LIBS} ${RAPTOR2_LIBS} ${TCLAP_LIBS} ${UUID_LIBS}
aff4imager_INCLUDES = ${RAPTOR2_CFLAGS} ${UUID_CFLAGS} ${TCLAP_CFLAGS} ${YAML_CPP_CFLAGS} ${ZLIB_CFLAGS}
aff4imager_LDFLAGS = $(STATIC_LINKERLDFLAGS)
aff4imager_SOURCES = aff4imager.cc

# Linux backwards compatibility shims for static builds.
if LINUX
aff4imager_SOURCES = aff4imager.cc compatibility.cc
else
aff4imager_SOURCES = aff4imager.cc
endif
@@ -166,6 +166,11 @@ class AFF4Object {
virtual void MarkDirty() {
_dirty = true;
}

private:
// AFF4Objects are not trivially copyable.
AFF4Object(const AFF4Object&) = delete;
AFF4Object& operator=(const AFF4Object&) = delete;
};


Oops, something went wrong.

0 comments on commit 077964a

Please sign in to comment.