Permalink
Browse files

Windows bug fixes. (#49)

* DataStore.Set() was changed to maintain multiple type
 attributes (to support aff4:Image simultaneously with
 aff4:ImageStream etc) but this just breaks other attributes. Now
 DataStore.Set() can specify if other attributes should be cleared. It
 really only makes sense to have the replace parameter. This only
 makes sense for AFF4_TYPE.

 * Fixed a bug with compressed/uncompressed chunks. AFF4 specifies
 that chunks which do not compress well should be written
 uncompressed.

 * More logging conversion.
  • Loading branch information...
scudette committed Dec 29, 2017
1 parent 546fdfe commit cd6d3eaf281d7105870971248254e2b0c1482bdc
Showing with 366 additions and 106 deletions.
  1. +69 −0 README.linux
  2. +124 −0 README.windows
  3. +9 −1 configure.ac
  4. +10 −9 src/aff4_directory.cc
  5. +4 −4 src/aff4_file.cc
  6. +22 −20 src/aff4_image.cc
  7. +21 −18 src/aff4_imager_utils.cc
  8. +17 −0 src/aff4_io.h
  9. +3 −2 src/aff4_map.cc
  10. +20 −7 src/data_store.cc
  11. +10 −5 src/data_store.h
  12. +1 −1 src/lexicon.cc
  13. +12 −17 src/libaff4.cc
  14. +10 −11 src/rdf.cc
  15. +34 −11 src/zip.cc
@@ -0,0 +1,69 @@
Building statically for linux.
==============================
The following explains how to build statically on Ubuntu 17.10.
apt-get install libtool-bin libboost-dev
# It is best to maintain a special prefix with static libraries so
# they do not pollute the system.
export PREFIX=/home/scudette/build/static/
export CXXFLAGS="-I$PREFIX/include"
apt-get source libraptor2-dev
cd raptor2-2.0.13/
./configure --prefix=$PREFIX --enable-static --without-www LDFLAGS="-L/home/scudette/build/static/lib -static -static-libstdc++" --enable-serializers="turtle ntriples" --enable-parsers="turtle ntriples"
make -j4 install
cd ..
apt-get source libpcre3
cd pcre3-8.39/
./configure --prefix=$PREFIX --enable-static LDFLAGS="-L$PREFIX/lib -static -static-libstdc++"
make -j4 install
cd ..
apt-get source libpcre++-dev
cd libpcre++-0.9.5/
./autogen.sh
cp /usr/bin/libtool ./libtool
./configure --prefix=$PREFIX --enable-static LDFLAGS="-L$PREFIX/lib -static -static-libstdc++"
make -j4 install
cd ..
apt-get source libyaml-cpp-dev
cd yaml-cpp-0.5.2/
cmake -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX . && make all install
cd ..
apt-get source libsnappy-dev
cd snappy-1.1.6/
./autogen.sh
./configure --prefix=$PREFIX --enable-static --without-gflags LDFLAGS="-L$PREFIX/lib -static -static-libstdc++"
make -j4 install
cd ..
apt-get source liburiparser-dev
cd uriparser-0.8.4/
./configure --prefix=$PREFIX --enable-static --without-gflags LDFLAGS="-L$PREFIX/lib -static -static-libstdc++" --disable-test --disable-doc
make -j4 install
cd ..
apt-get source libspdlog-dev
cd spdlog-0.11.0/
cmake -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX . && make all install
cd ..
git clone https://github.com/google/aff4.git
cd aff4
./autogen.sh
./configure --prefix=$PREFIX --enable-static LDFLAGS="-L$PREFIX/lib -L$PREFIX/libexec" --enable-static-binaries
make -j4 install
cd ..
git clone https://github.com/google/rekall.git
cd rekall/tools/pmem
./bootstrap
./configure --prefix=$PREFIX --enable-static LDFLAGS="-L$PREFIX/lib -L$PREFIX/libexec" --enable-static-binaries
make -j4 install
cd ..
@@ -0,0 +1,124 @@
Building for windows.
=====================
The following instructions are for building windows binaries, on a
Linux system. On Ubuntu systems one must install the mingw platform
first (using apt-get install g++-mingw-w64 mingw-w64).
This installs several flavors of compilers. We need the posix version
so you will need to switch via ubuntu's alternatives version:
sudo update-alternatives --config i686-w64-mingw32-g++
sudo update-alternatives --config i686-w64-mingw32-gcc
and select /usr/bin/i686-w64-mingw32-g++-posix and /usr/bin/i686-w64-mingw32-gcc-posix
# It is best to maintain a special prefix with static libraries so
# they do not pollute the system.
export PREFIX=/home/scudette/build/mingw/
export CXXFLAGS="-I$PREFIX/include"
export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig/"
apt-get source zlib1g
cd zlib-1.2.11.dfsg/
make -f win32/Makefile.gcc PREFIX=i686-w64-mingw32-
make -f win32/Makefile.gcc install INCLUDE_PATH=$PREFIX/include LIBRARY_PATH=$PREFIX/lib BINARY_PATH==$PREFIX/bin
cd ..
apt-get source libraptor2-dev
cd raptor2-2.0.14/
./configure --prefix=$PREFIX --enable-static --without-www LDFLAGS="-L$PREFIX/lib -static -static-libstdc++" \
--host=i686-w64-mingw32 --enable-parsers="turtle ntriples" --enable-serializers="turtle ntriples"
make -j4 install
cd ..
apt-get source libtclap-dev
cd tclap-1.2.1/
./configure --prefix=$PREFIX --enable-static LDFLAGS="-L$PREFIX/lib -static -static-libstdc++" --host=i686-w64-mingw32 \
CXXFLAGS="-static -mnop-fun-dllimport -DPCRE_STATIC"
make -j4 install
cd ..
apt-get source pcre3
cd pcre3-8.39/
./configure --prefix=$PREFIX --enable-static LDFLAGS="-L$PREFIX/lib -static -static-libstdc++" --host=i686-w64-mingw32 \
CFLAGS="-static -mnop-fun-dllimport" --disable-cpp
cp /usr/bin/libtool ./libtool
make -j4 install
cd ..
apt-get source libpcre++-dev
cd libpcre++-0.9.5/
./autogen.sh
# The libtool that comes with it is broken and will cause a crash -
# copy the one from the system.
cp /usr/bin/libtool ./libtool
./configure --prefix=$PREFIX --enable-static LDFLAGS="-L$PREFIX/lib -static -static-libstdc++" --host=i686-w64-mingw32 \
CXXFLAGS="-static -mnop-fun-dllimport -DPCRE_STATIC" --with-pcre-include="$PREFIX/include/"
make -j4 install
cd ..
apt-get source libsnappy-dev
cd snappy-1.1.6/
./configure --prefix=$PREFIX --enable-static LDFLAGS="-L$PREFIX/lib -static -static-libstdc++" --host=i686-w64-mingw32 \
CXXFLAGS="-static -mnop-fun-dllimport -I$PREFIX/include/"
# Unit test will fail to build.
make -j4 install
cd ..
apt-get source liburiparser-dev
cd uriparser-0.8.4/
./configure --prefix=$PREFIX --enable-static LDFLAGS="-L$PREFIX/lib -static -static-libstdc++" --host=i686-w64-mingw32 \
CXXFLAGS="-static -mnop-fun-dllimport -I$PREFIX/include/" --disable-test --disable-doc
# This insists on installing a binary which is not compatible with
# windows. Remove all code from tool/uriparse.c except for main().
make -j4 install
cd ..
apt-get source libyaml-cpp-dev
cd yaml-cpp-0.5.2/
echo "
# the name of the target operating system
SET(CMAKE_SYSTEM_NAME Windows)
# which compilers to use for C and C++
SET(CMAKE_C_COMPILER i686-w64-mingw32-gcc)
SET(CMAKE_CXX_COMPILER i686-w64-mingw32-g++)
SET(CMAKE_RC_COMPILER i686-w64-mingw32-windres)
# here is the target environment located
SET(CMAKE_FIND_ROOT_PATH /usr/i686-w64-mingw32 $PREFIX )
# adjust the default behaviour of the FIND_XXX() commands:
# search headers and libraries in the target environment, search
# programs in the host environment
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
" > Toolchain-mingw32.cmake
cmake -DCMAKE_TOOLCHAIN_FILE=Toolchain-mingw32.cmake -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX .
make install -j4
cd ..
git clone https://github.com/gabime/spdlog.git
cd spdlog/
cmake -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX . && make install -j4
cd ..
cd aff4
./configure --prefix=$PREFIX --enable-static LDFLAGS="-L$PREFIX/lib -static -static-libstdc++" --host=i686-w64-mingw32 \
CXXFLAGS="-static -mnop-fun-dllimport -I$PREFIX/include/" --enable-static-binaries
make -j4 install
cd ..
On very old versions of windows the binary might fail with a message
“unable to find strerror_s in msvcrt.dll”. This is because such old
versions do not contain the function strerror_s, but they do contain
strerror. The easiest way to fix this is to hexedit the import table
of the binary - simply find the string strerror_s and replace the _s
with null bytes. This makes the same binary work fine on older windows
versions.
@@ -58,7 +58,7 @@ fi
# Checks for libraries.
PKG_CHECK_MODULES([RAPTOR2], [raptor2], [], [AC_MSG_ERROR([raptor RDF library (libraptor2-dev) not found])])
PKG_CHECK_MODULES([UUID], [uuid], [AC_DEFINE([HAVE_LIBUUID], [1], ["Have UUID"])], [AC_MSG_ERROR([uuid library (uuid-dev) not found])])
PKG_CHECK_MODULES([ZLIB], [zlib], [], [AC_MSG_ERROR([zlib library (zlib1g-dev) not found])])
AC_CHECK_LIB([pcre++], [main], [], [AC_MSG_ERROR([pcre++ library (libpcre++-dev) not found])])
AC_CHECK_LIB([snappy], [main], [], [AC_MSG_ERROR([Google Snappy Compression library (libsnappy-dev) not found])])
@@ -100,6 +100,14 @@ AM_CONDITIONAL([GCC], test "$GCC" = yes) # let the Makefile know if we're gcc
AM_CONDITIONAL([STATIC_BUILD], [test x$enable_static_binaries = xyes])
AM_CONDITIONAL([WIN32_NATIVE_HOST], [test x$mingw_cv_win32_host = xyes])
# UUID support is not required on windows but mandatory on unix.
if test "x$mingw_cv_win32_host" = "xno"; then
PKG_CHECK_MODULES(
[UUID], [uuid], [AC_DEFINE([HAVE_LIBUUID], [1], ["Have UUID"])],
[AC_MSG_ERROR([uuid library (uuid-dev) not found])])
fi
AC_SUBST(LIBTOOL_DEPS)
AC_SUBST(SO_VERSION)
@@ -57,7 +57,8 @@ AFF4ScopedPtr<AFF4Directory> AFF4Directory::NewAFF4Directory(
}
}
resolver->Set(result->urn, AFF4_TYPE, new URN(AFF4_DIRECTORY_TYPE));
resolver->Set(result->urn, AFF4_TYPE, new URN(AFF4_DIRECTORY_TYPE),
/* replace= */ false);
resolver->Set(result->urn, AFF4_STORED, new URN(root_urn));
if (result) {
@@ -123,7 +124,8 @@ AFF4Status AFF4Directory::LoadFromURN() {
urn.Set(urn_string);
// Set these triples with the new URN so we know how to open it.
resolver->Set(urn, AFF4_TYPE, new URN(AFF4_DIRECTORY_TYPE));
resolver->Set(urn, AFF4_TYPE, new URN(AFF4_DIRECTORY_TYPE),
/* replace= */ false);
resolver->Set(urn, AFF4_STORED, new URN(storage));
resolver->logger->info("AFF4Directory volume found: {}", urn);
@@ -228,9 +230,10 @@ bool AFF4Directory::IsDirectory(const URN& urn) {
// path: Absolute path of the directory that will be deleted
// The path must not be terminated with a path separator.
AFF4Status AFF4Directory::RemoveDirectory(DataStore *resolver, const string& path) {
AFF4Status AFF4Directory::RemoveDirectory(DataStore *resolver,
const std::string& path) {
WIN32_FIND_DATA ffd;
string search_str = path + PATH_SEP_STR + "*";
std::string search_str = path + PATH_SEP_STR + "*";
HANDLE hFind = INVALID_HANDLE_VALUE;
hFind = FindFirstFile(search_str.c_str(), &ffd);
@@ -252,7 +255,7 @@ AFF4Status AFF4Directory::RemoveDirectory(DataStore *resolver, const string& pat
return result;
}
} else {
string filename = path + PATH_SEP_STR + ffd.cFileName;
std::string filename = path + PATH_SEP_STR + ffd.cFileName;
resolver->logger->info("Deleting file {}", filename);
if (!::DeleteFile(filename.c_str())) {
resolver->logger->error("Failed: {}", GetLastErrorMessage());
@@ -272,7 +275,7 @@ AFF4Status AFF4Directory::RemoveDirectory(DataStore *resolver, const string& pat
return STATUS_OK;
}
AFF4Status AFF4Directory::MkDir(DataStore* resolver, const string& path) {
AFF4Status AFF4Directory::MkDir(DataStore* resolver, const std::string& path) {
resolver->logger->info("MkDir {}", path);
if (!CreateDirectory(path.c_str(), nullptr)) {
@@ -289,7 +292,7 @@ AFF4Status AFF4Directory::MkDir(DataStore* resolver, const string& path) {
return STATUS_OK;
}
bool AFF4Directory::IsDirectory(const string& filename) {
bool AFF4Directory::IsDirectory(const std::string& filename) {
DWORD dwAttrib = GetFileAttributes(filename.c_str());
bool result = (dwAttrib != INVALID_FILE_ATTRIBUTES &&
@@ -300,8 +303,6 @@ bool AFF4Directory::IsDirectory(const string& filename) {
char last = *(filename.rbegin());
result |= (last == '/' || last == '\\');
resolver->logger->info("IsDirectory {}: {}", filename, result);
return result;
}
@@ -101,7 +101,7 @@ AFF4Status FileBackedObject::LoadFromURN() {
resolver->logger->info("Opening file {}", filename);
vector<string> directory_components = split(filename, PATH_SEP);
std::vector<std::string> directory_components = split(filename, PATH_SEP);
directory_components.pop_back();
// Attribute is optional so if it is not there we just go with false.
@@ -179,9 +179,9 @@ AFF4Status FileBackedObject::LoadFromURN() {
return STATUS_OK;
}
string FileBackedObject::Read(size_t length) {
std::string FileBackedObject::Read(size_t length) {
DWORD buffer_size = length;
unique_ptr<char[]> result(new char[length]);
std::unique_ptr<char[]> result(new char[length]);
if (properties.seekable) {
LARGE_INTEGER tmp;
@@ -200,7 +200,7 @@ string FileBackedObject::Read(size_t length) {
readptr += buffer_size;
return string(result.get(), buffer_size);
return std::string(result.get(), buffer_size);
}
Oops, something went wrong.

0 comments on commit cd6d3ea

Please sign in to comment.