i#4318 xarch memtrace: Support gzipped .raw files (#4332)

Adds a new gzip wrapper around std::istream to support compressing
.raw files.  This lets us easily shrink checked-in test files, and
aids other relocate-then-process situations with raw files.

Updates the altbin test by compressing its .raw files.

Issue: #4318

clients/drcachesim/common/gzip_istream.h

@@ -0,0 +1,114 @@
+/* **********************************************************
+ * Copyright (c) 2018-2020 Google, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Google, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* gzip_istream_t: a wrapper around zlib gzFile to match the parts of the
+ * std::istream interface we use for raw2trace and file_reader_t.
+ * Supports only limited seeking within the current internal buffer.
+ */
+
+#ifndef _GZIP_ISTREAM_H_
+#define _GZIP_ISTREAM_H_ 1
+
+#ifndef HAS_ZLIB
+#    error HAS_ZLIB is required
+#endif
+#include <fstream>
+#include <zlib.h>
+
+/* We need to override the stream buffer class which is where the file
+ * reads happen.  The stream buffer base class reads from eback()..egptr()
+ * with the next to read at gptr().
+ */
+class gzip_istreambuf_t : public std::basic_streambuf<char, std::char_traits<char>> {
+public:
+    gzip_istreambuf_t(const std::string &path)
+    {
+        file_ = gzopen(path.c_str(), "rb");
+        if (file_ != nullptr) {
+            buf_ = new char[buffer_size_];
+        }
+    }
+    ~gzip_istreambuf_t() override
+    {
+        delete[] buf_;
+        if (file_ != nullptr)
+            gzclose(file_);
+    }
+    int
+    underflow() override
+    {
+        if (file_ == nullptr)
+            return traits_type::eof();
+        if (gptr() == egptr()) {
+            int len = gzread(file_, buf_, buffer_size_);
+            if (len <= 0)
+                return traits_type::eof();
+            setg(buf_, buf_, buf_ + len);
+        }
+        return *gptr();
+    }
+    std::iostream::pos_type
+    seekoff(std::iostream::off_type off, std::ios_base::seekdir dir,
+            std::ios_base::openmode which = std::ios_base::in) override
+    {
+        if (dir == std::ios_base::cur &&
+            ((off >= 0 && gptr() + off < egptr()) ||
+             (off < 0 && gptr() + off >= eback())))
+            gbump(off);
+        else {
+            // Unsupported!
+            return -1;
+        }
+        return gptr() - eback();
+    }
+
+private:
+    static const int buffer_size_ = 4096;
+    gzFile file_ = nullptr;
+    char *buf_ = nullptr;
+};
+
+class gzip_istream_t : public std::istream {
+public:
+    explicit gzip_istream_t(const std::string &path)
+        : std::istream(new gzip_istreambuf_t(path))
+    {
+        if (!rdbuf())
+            setstate(std::ios::badbit);
+    }
+    virtual ~gzip_istream_t() override
+    {
+        delete rdbuf();
+    }
+};
+
+#endif /* _GZIP_ISTREAM_H_ */

clients/drcachesim/tests/offline-altbindir.templatex

@@ -5,7 +5,7 @@ Opcode mix tool results:
           15165 :       stp
 .*
 #elif defined(X86) && defined(X64)
-ERROR: failed to initialize analyzer: Directory setup failed: Failed sanity checks for thread log file .*/drmemtrace.threadsig.aarch64/raw/drmemtrace.threadsig..*.raw: Architecture mismatch: trace recorded on aarch64 but tools built for x86_64
+ERROR: failed to initialize analyzer: Directory setup failed: Failed sanity checks for thread log file .*/drmemtrace.threadsig.aarch64/raw/drmemtrace.threadsig..*.raw.gz: Architecture mismatch: trace recorded on aarch64 but tools built for x86_64
 #else
 ERROR: failed to initialize analyzer: Failed to create analysis tool: Tool failed to initialize: Failed to load binaries: Failed to map module /tmp/nonexistent/threadsig
 #endif

clients/drcachesim/tracer/raw2trace.h

@@ -61,6 +61,9 @@
 #endif
 
 #define OUTFILE_SUFFIX "raw"
+#ifdef HAS_ZLIB
+#    define OUTFILE_SUFFIX_GZ "gz"
+#endif
 #define OUTFILE_SUBDIR "raw"
 #define TRACE_SUBDIR "trace"
 #ifdef HAS_ZLIB

clients/drcachesim/tracer/raw2trace_directory.cpp

@@ -49,6 +49,7 @@
 #    include <windows.h>
 #endif
 #ifdef HAS_ZLIB
+#    include "common/gzip_istream.h"
 #    include "common/gzip_ostream.h"
 #endif
 
@@ -108,17 +109,35 @@ raw2trace_directory_t::open_thread_log_file(const char *basename)
         strcmp(basename, DRMEMTRACE_FUNCTION_LIST_FILENAME) == 0)
         return "";
     // Skip any non-.raw in case someone put some other file in there.
-    const char *basename_pre_suffix = strrchr(basename, '.');
-    if (basename_pre_suffix != nullptr)
-        basename_pre_suffix = strstr(basename_pre_suffix, OUTFILE_SUFFIX);
+    const char *basename_dot = strrchr(basename, '.');
+    if (basename_dot == nullptr)
+        return "";
+    const char *basename_pre_suffix = nullptr;
+    bool is_gzipped = false;
+#ifdef HAS_ZLIB
+    basename_pre_suffix = strstr(basename_dot, OUTFILE_SUFFIX_GZ);
+    if (basename_pre_suffix != nullptr) {
+        is_gzipped = true;
+        basename_dot = strrchr(basename_pre_suffix, '.');
+    }
+#endif
+    if (basename_pre_suffix == nullptr)
+        basename_pre_suffix = strstr(basename_dot, OUTFILE_SUFFIX);
     if (basename_pre_suffix == nullptr)
         return "";
     if (dr_snprintf(path, BUFFER_SIZE_ELEMENTS(path), "%s%s%s", indir_.c_str(), DIRSEP,
                     basename) <= 0) {
         return "Failed to get full path of file " + std::string(basename);
     }
     NULL_TERMINATE_BUFFER(path);
-    in_files_.push_back(new std::ifstream(path, std::ifstream::binary));
+    std::istream *ifile;
+#ifdef HAS_ZLIB
+    if (is_gzipped)
+        ifile = new gzip_istream_t(path);
+#endif
+    if (!is_gzipped)
+        ifile = new std::ifstream(path, std::ifstream::binary);
+    in_files_.push_back(ifile);
     if (!(*in_files_.back()))
         return "Failed to open thread log file " + std::string(path);
     std::string error = raw2trace_t::check_thread_file(in_files_.back());

suite/tests/CMakeLists.txt

@@ -3348,7 +3348,8 @@ endif ()
     # Test postprocessing and analysis with an alternate binary directory.
     # Limited to UNIX because the checked-in modules.log is in UNIX format
     # (Windows needs extra fields per library).
-    if (UNIX)
+    # Limited to ZLIB_FOUND because we test gzipped .raw files here as well.
+    if (UNIX AND ZLIB_FOUND)
       # We make a writable copy so we can both write the final trace there and run our
       # analyzer in a single command line.
       set(srcdir