[Reproducers] Capture and replay interpreter commands.

This patch adds the necessary logic to capture and replay commands
entered into the command interpreter. A DataRecorder shadows the input
and writes its data to a know file. During replay this file is used as
the command interpreter's input.

It's possible to the command interpreter more than once, with a
different input source. We support this scenario by using multiple
buffers. The synchronization for this takes place at the SB layer, where
we create a new recorder every time the debugger input is changed.
During replay we use the corresponding buffer as input.

Differential revision: https://reviews.llvm.org/D58564

llvm-svn: 355249

lldb/include/lldb/Core/Debugger.h

@@ -63,6 +63,11 @@ class SymbolContext;
 namespace lldb_private {
 class Target;
 }
+namespace lldb_private {
+namespace repro {
+class DataRecorder;
+}
+} // namespace lldb_private
 namespace llvm {
 class raw_ostream;
 }
@@ -129,7 +134,10 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   lldb::StreamFileSP GetErrorFile() { return m_error_file_sp; }
 
-  void SetInputFileHandle(FILE *fh, bool tranfer_ownership);
+  repro::DataRecorder *GetInputRecorder();
+
+  void SetInputFileHandle(FILE *fh, bool tranfer_ownership,
+                          repro::DataRecorder *recorder = nullptr);
 
   void SetOutputFileHandle(FILE *fh, bool tranfer_ownership);
 
@@ -370,6 +378,9 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
   lldb::StreamFileSP m_output_file_sp;
   lldb::StreamFileSP m_error_file_sp;
 
+  /// Used for shadowing the input file when capturing a reproducer.
+  repro::DataRecorder *m_input_recorder;
+
   lldb::BroadcasterManagerSP m_broadcaster_manager_sp; // The debugger acts as a
                                                        // broadcaster manager of
                                                        // last resort.

lldb/include/lldb/Core/IOHandler.h

@@ -13,6 +13,7 @@
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/Flags.h"
 #include "lldb/Utility/Predicate.h"
+#include "lldb/Utility/Reproducer.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StringList.h"
 #include "lldb/lldb-defines.h"
@@ -58,7 +59,8 @@ class IOHandler {
   IOHandler(Debugger &debugger, IOHandler::Type type,
             const lldb::StreamFileSP &input_sp,
             const lldb::StreamFileSP &output_sp,
-            const lldb::StreamFileSP &error_sp, uint32_t flags);
+            const lldb::StreamFileSP &error_sp, uint32_t flags,
+            repro::DataRecorder *data_recorder);
 
   virtual ~IOHandler();
 
@@ -169,6 +171,7 @@ class IOHandler {
   lldb::StreamFileSP m_input_sp;
   lldb::StreamFileSP m_output_sp;
   lldb::StreamFileSP m_error_sp;
+  repro::DataRecorder *m_data_recorder;
   Predicate<bool> m_popped;
   Flags m_flags;
   Type m_type;
@@ -343,7 +346,8 @@ class IOHandlerEditline : public IOHandler {
                     uint32_t line_number_start, // If non-zero show line numbers
                                                 // starting at
                                                 // 'line_number_start'
-                    IOHandlerDelegate &delegate);
+                    IOHandlerDelegate &delegate,
+                    repro::DataRecorder *data_recorder);
 
   IOHandlerEditline(Debugger &debugger, IOHandler::Type type,
                     const lldb::StreamFileSP &input_sp,
@@ -355,7 +359,8 @@ class IOHandlerEditline : public IOHandler {
                     uint32_t line_number_start, // If non-zero show line numbers
                                                 // starting at
                                                 // 'line_number_start'
-                    IOHandlerDelegate &delegate);
+                    IOHandlerDelegate &delegate,
+                    repro::DataRecorder *data_recorder);
 
   IOHandlerEditline(Debugger &, IOHandler::Type, const char *, const char *,
                     const char *, bool, bool, uint32_t,

lldb/include/lldb/Utility/Reproducer.h

@@ -111,6 +111,50 @@ class FileProvider : public Provider<FileProvider> {
   FileCollector m_collector;
 };
 
+class DataRecorder {
+public:
+  DataRecorder(FileSpec filename, std::error_code &ec)
+      : m_filename(std::move(filename)),
+        m_os(m_filename.GetPath(), ec, llvm::sys::fs::F_Text) {}
+
+  static llvm::Expected<std::unique_ptr<DataRecorder>>
+  Create(FileSpec filename);
+
+  template <typename T> void Record(const T &t, bool newline = false) {
+    m_os << t;
+    if (newline)
+      m_os << '\n';
+  }
+
+  const FileSpec &GetFilename() { return m_filename; }
+
+private:
+  FileSpec m_filename;
+  llvm::raw_fd_ostream m_os;
+};
+
+struct CommandInfo {
+  static const char *name;
+  static const char *file;
+};
+
+class CommandProvider : public Provider<CommandProvider> {
+public:
+  typedef CommandInfo info;
+
+  CommandProvider(const FileSpec &directory) : Provider(directory) {}
+
+  DataRecorder *GetNewDataRecorder();
+
+  void Keep() override;
+  void Discard() override;
+
+  static char ID;
+
+private:
+  std::vector<std::unique_ptr<DataRecorder>> m_data_recorders;
+};
+
 /// The generator is responsible for the logic needed to generate a
 /// reproducer. For doing so it relies on providers, who serialize data that
 /// is necessary for reproducing  a failure.

lldb/source/API/SBDebugger.cpp

@@ -57,6 +57,45 @@
 using namespace lldb;
 using namespace lldb_private;
 
+/// Helper class for replaying commands through the reproducer.
+class CommandLoader {
+public:
+  CommandLoader(std::vector<std::string> files) : m_files(files) {}
+
+  static std::unique_ptr<CommandLoader> Create() {
+    repro::Loader *loader = repro::Reproducer::Instance().GetLoader();
+    if (!loader)
+      return {};
+
+    FileSpec file = loader->GetFile<repro::CommandInfo>();
+    if (!file)
+      return {};
+
+    auto error_or_file = llvm::MemoryBuffer::getFile(file.GetPath());
+    if (auto err = error_or_file.getError())
+      return {};
+
+    std::vector<std::string> files;
+    llvm::yaml::Input yin((*error_or_file)->getBuffer());
+    yin >> files;
+
+    if (auto err = yin.error())
+      return {};
+
+    return llvm::make_unique<CommandLoader>(std::move(files));
+  }
+
+  FILE *GetNextFile() {
+    if (m_index >= m_files.size())
+      return nullptr;
+    return FileSystem::Instance().Fopen(m_files[m_index++].c_str(), "r");
+  }
+
+private:
+  std::vector<std::string> m_files;
+  unsigned m_index = 0;
+};
+
 static llvm::sys::DynamicLibrary LoadPlugin(const lldb::DebuggerSP &debugger_sp,
                                             const FileSpec &spec,
                                             Status &error) {
@@ -269,8 +308,18 @@ void SBDebugger::SetInputFileHandle(FILE *fh, bool transfer_ownership) {
         static_cast<void *>(m_opaque_sp.get()), static_cast<void *>(fh),
         transfer_ownership);
 
-  if (m_opaque_sp)
-    m_opaque_sp->SetInputFileHandle(fh, transfer_ownership);
+  if (!m_opaque_sp)
+    return;
+
+  repro::DataRecorder *recorder = nullptr;
+  if (repro::Generator *g = repro::Reproducer::Instance().GetGenerator())
+    recorder = g->GetOrCreate<repro::CommandProvider>().GetNewDataRecorder();
+
+  static std::unique_ptr<CommandLoader> loader = CommandLoader::Create();
+  if (loader)
+    fh = loader->GetNextFile();
+
+  m_opaque_sp->SetInputFileHandle(fh, transfer_ownership, recorder);
 }
 
 void SBDebugger::SetOutputFileHandle(FILE *fh, bool transfer_ownership) {

lldb/source/Commands/CommandObjectCommands.cpp

@@ -1040,7 +1040,7 @@ a number follows 'f':"
           llvm::StringRef(),     // Continuation prompt
           multiple_lines, color_prompt,
           0, // Don't show line numbers
-          *this));
+          *this, nullptr));
 
       if (io_handler_sp) {
         debugger.PushIOHandler(io_handler_sp);

lldb/source/Commands/CommandObjectExpression.cpp

@@ -234,7 +234,7 @@ Single and multi-line expressions:
 with no newlines.  To evaluate a multi-line expression, \
 hit a return after an empty expression, and lldb will enter the multi-line expression editor. \
 Hit return on an empty line to end the multi-line expression."
-      
+
       R"(
 
 Timeouts:
@@ -560,7 +560,7 @@ void CommandObjectExpression::GetMultilineExpression() {
                             llvm::StringRef(), // Continuation prompt
                             multiple_lines, color_prompt,
                             1, // Show line numbers starting at 1
-                            *this));
+                            *this, nullptr));
 
   StreamFileSP output_sp(io_handler_sp->GetOutputStreamFile());
   if (output_sp) {

lldb/source/Core/Debugger.cpp

@@ -760,6 +760,7 @@ Debugger::Debugger(lldb::LogOutputCallback log_callback, void *baton)
       m_input_file_sp(std::make_shared<StreamFile>(stdin, false)),
       m_output_file_sp(std::make_shared<StreamFile>(stdout, false)),
       m_error_file_sp(std::make_shared<StreamFile>(stderr, false)),
+      m_input_recorder(nullptr),
       m_broadcaster_manager_sp(BroadcasterManager::MakeBroadcasterManager()),
       m_terminal_state(), m_target_list(*this), m_platform_list(),
       m_listener_sp(Listener::MakeListener("lldb.Debugger")),
@@ -877,7 +878,11 @@ void Debugger::SetAsyncExecution(bool async_execution) {
   m_command_interpreter_up->SetSynchronous(!async_execution);
 }
 
-void Debugger::SetInputFileHandle(FILE *fh, bool tranfer_ownership) {
+repro::DataRecorder *Debugger::GetInputRecorder() { return m_input_recorder; }
+
+void Debugger::SetInputFileHandle(FILE *fh, bool tranfer_ownership,
+                                  repro::DataRecorder *recorder) {
+  m_input_recorder = recorder;
   if (m_input_file_sp)
     m_input_file_sp->GetFile().SetStream(fh, tranfer_ownership);
   else

lldb/source/Core/IOHandler.cpp

@@ -76,16 +76,19 @@ IOHandler::IOHandler(Debugger &debugger, IOHandler::Type type)
                 StreamFileSP(), // Adopt STDIN from top input reader
                 StreamFileSP(), // Adopt STDOUT from top input reader
                 StreamFileSP(), // Adopt STDERR from top input reader
-                0)              // Flags
-{}
+                0,              // Flags
+                nullptr         // Shadow file recorder
+      ) {}
 
 IOHandler::IOHandler(Debugger &debugger, IOHandler::Type type,
                      const lldb::StreamFileSP &input_sp,
                      const lldb::StreamFileSP &output_sp,
-                     const lldb::StreamFileSP &error_sp, uint32_t flags)
+                     const lldb::StreamFileSP &error_sp, uint32_t flags,
+                     repro::DataRecorder *data_recorder)
     : m_debugger(debugger), m_input_sp(input_sp), m_output_sp(output_sp),
-      m_error_sp(error_sp), m_popped(false), m_flags(flags), m_type(type),
-      m_user_data(nullptr), m_done(false), m_active(false) {
+      m_error_sp(error_sp), m_data_recorder(data_recorder), m_popped(false),
+      m_flags(flags), m_type(type), m_user_data(nullptr), m_done(false),
+      m_active(false) {
   // If any files are not specified, then adopt them from the top input reader.
   if (!m_input_sp || !m_output_sp || !m_error_sp)
     debugger.AdoptTopIOHandlerFilesIfInvalid(m_input_sp, m_output_sp,
@@ -153,7 +156,7 @@ IOHandlerConfirm::IOHandlerConfirm(Debugger &debugger, llvm::StringRef prompt,
           llvm::StringRef(), // No continuation prompt
           false,             // Multi-line
           false, // Don't colorize the prompt (i.e. the confirm message.)
-          0, *this),
+          0, *this, nullptr),
       m_default_response(default_response), m_user_response(default_response) {
   StreamString prompt_stream;
   prompt_stream.PutCString(prompt);
@@ -264,15 +267,15 @@ IOHandlerEditline::IOHandlerEditline(
     const char *editline_name, // Used for saving history files
     llvm::StringRef prompt, llvm::StringRef continuation_prompt,
     bool multi_line, bool color_prompts, uint32_t line_number_start,
-    IOHandlerDelegate &delegate)
+    IOHandlerDelegate &delegate, repro::DataRecorder *data_recorder)
     : IOHandlerEditline(debugger, type,
                         StreamFileSP(), // Inherit input from top input reader
                         StreamFileSP(), // Inherit output from top input reader
                         StreamFileSP(), // Inherit error from top input reader
                         0,              // Flags
                         editline_name,  // Used for saving history files
                         prompt, continuation_prompt, multi_line, color_prompts,
-                        line_number_start, delegate) {}
+                        line_number_start, delegate, data_recorder) {}
 
 IOHandlerEditline::IOHandlerEditline(
     Debugger &debugger, IOHandler::Type type,
@@ -281,8 +284,9 @@ IOHandlerEditline::IOHandlerEditline(
     const char *editline_name, // Used for saving history files
     llvm::StringRef prompt, llvm::StringRef continuation_prompt,
     bool multi_line, bool color_prompts, uint32_t line_number_start,
-    IOHandlerDelegate &delegate)
-    : IOHandler(debugger, type, input_sp, output_sp, error_sp, flags),
+    IOHandlerDelegate &delegate, repro::DataRecorder *data_recorder)
+    : IOHandler(debugger, type, input_sp, output_sp, error_sp, flags,
+                data_recorder),
 #ifndef LLDB_DISABLE_LIBEDIT
       m_editline_up(),
 #endif
@@ -338,7 +342,10 @@ void IOHandlerEditline::Deactivate() {
 bool IOHandlerEditline::GetLine(std::string &line, bool &interrupted) {
 #ifndef LLDB_DISABLE_LIBEDIT
   if (m_editline_up) {
-    return m_editline_up->GetLine(line, interrupted);
+    bool b = m_editline_up->GetLine(line, interrupted);
+    if (m_data_recorder)
+      m_data_recorder->Record(line, true);
+    return b;
   } else {
 #endif
     line.clear();
@@ -394,6 +401,8 @@ bool IOHandlerEditline::GetLine(std::string &line, bool &interrupted) {
         }
       }
       m_editing = false;
+      if (m_data_recorder && got_line)
+        m_data_recorder->Record(line, true);
       // We might have gotten a newline on a line by itself make sure to return
       // true in this case.
       return got_line;

lldb/source/Expression/REPL.cpp

@@ -75,13 +75,13 @@ lldb::IOHandlerSP REPL::GetIOHandler() {
     Debugger &debugger = m_target.GetDebugger();
     m_io_handler_sp = std::make_shared<IOHandlerEditline>(
         debugger, IOHandler::Type::REPL,
-                              "lldb-repl", // Name of input reader for history
-                              llvm::StringRef("> "), // prompt
-                              llvm::StringRef(". "), // Continuation prompt
-                              true,                  // Multi-line
-                              true, // The REPL prompt is always colored
-                              1,    // Line number
-        *this);
+        "lldb-repl",           // Name of input reader for history
+        llvm::StringRef("> "), // prompt
+        llvm::StringRef(". "), // Continuation prompt
+        true,                  // Multi-line
+        true,                  // The REPL prompt is always colored
+        1,                     // Line number
+        *this, nullptr);
 
     // Don't exit if CTRL+C is pressed
     static_cast<IOHandlerEditline *>(m_io_handler_sp.get())

lldb/source/Interpreter/CommandInterpreter.cpp

@@ -2483,7 +2483,7 @@ void CommandInterpreter::HandleCommandsFromFile(
                // or written
       debugger.GetPrompt(), llvm::StringRef(),
       false, // Not multi-line
-      debugger.GetUseColor(), 0, *this));
+      debugger.GetUseColor(), 0, *this, nullptr));
   const bool old_async_execution = debugger.GetAsyncExecution();
 
   // Set synchronous execution if we are not stopping on continue
@@ -2905,8 +2905,9 @@ void CommandInterpreter::GetLLDBCommandsFromIOHandler(
                             llvm::StringRef(), // Continuation prompt
                             true,              // Get multiple lines
                             debugger.GetUseColor(),
-                            0,          // Don't show line numbers
-                            delegate)); // IOHandlerDelegate
+                            0,         // Don't show line numbers
+                            delegate,  // IOHandlerDelegate
+                            nullptr)); // FileShadowCollector
 
   if (io_handler_sp) {
     io_handler_sp->SetUserData(baton);
@@ -2928,8 +2929,9 @@ void CommandInterpreter::GetPythonCommandsFromIOHandler(
                             llvm::StringRef(), // Continuation prompt
                             true,              // Get multiple lines
                             debugger.GetUseColor(),
-                            0,          // Don't show line numbers
-                            delegate)); // IOHandlerDelegate
+                            0,         // Don't show line numbers
+                            delegate,  // IOHandlerDelegate
+                            nullptr)); // FileShadowCollector
 
   if (io_handler_sp) {
     io_handler_sp->SetUserData(baton);
@@ -2980,8 +2982,9 @@ CommandInterpreter::GetIOHandler(bool force_create,
         llvm::StringRef(), // Continuation prompt
         false, // Don't enable multiple line input, just single line commands
         m_debugger.GetUseColor(),
-        0, // Don't show line numbers
-        *this);
+        0,     // Don't show line numbers
+        *this, // IOHandlerDelegate
+        GetDebugger().GetInputRecorder());
   }
   return m_command_io_handler_sp;
 }