Response files

doc/manual.asciidoc

@@ -498,6 +498,26 @@ aborting due to a missing input.
  This may cause the output's reverse dependencies to be removed from the
  list of pending build actions.
 
+`rspfile`
+`rspfile_content`:: if present (both), Ninja will use a response file
+  for the given command, i.e. write the selected string (`rspfile_content`)
+  to the given file (`rspfile`) before calling the command and delete
+  the file after successful execution of the command.
++
+This is particularly useful on Windows OS, where the maximal length of 
+a command line is limited and response files must be used instead.
++
+Use it like in the following example:
++
+----
+rule link
+  command = link.exe /OUT$out [usual link flags here] @$out.rsp
+  rspfile = $out.rsp
+  rspfile_content = $in
+
+build myapp.exe: link a.obj b.obj [possibly many other .obj files]
+----
+
 Additionally, the special `$in` and `$out` variables expand to the
 space-separated list of files provided to the `build` line referencing
 this `rule`.

src/build.cc

@@ -343,8 +343,8 @@ void Plan::CleanNode(BuildLog* build_log, Node* node) {
       for (vector<Node*>::iterator ni = begin; ni != end; ++ni)
         if ((*ni)->mtime() > most_recent_input)
           most_recent_input = (*ni)->mtime();
-      string command = (*ei)->EvaluateCommand();
-
+      string command = (*ei)->EvaluateCommand(true);
+      
       // Now, recompute the dirty state of each output.
       bool all_outputs_clean = true;
       for (vector<Node*>::iterator ni = (*ei)->outputs_.begin();
@@ -575,6 +575,13 @@ bool Builder::StartEdge(Edge* edge, string* err) {
     if (!disk_interface_->MakeDirs((*i)->path()))
       return false;
   }
+
+  // Create response file, if needed
+  // XXX: this may also block; do we care?
+  if (edge->HasRspFile()) {
+    if (!disk_interface_->WriteFile(edge->GetRspFile(), edge->GetRspFileContent())) 
+      return false;
+  }
 
   // start command computing and run it
   if (!command_runner_->StartCommand(edge)) {
@@ -632,6 +639,10 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) {
       }
     }
 
+    // delete the response file on success (if exists)
+    if (edge->HasRspFile()) 
+      disk_interface_->RemoveFile(edge->GetRspFile());
+
     plan_.EdgeFinished(edge);
   }
 

src/build_log.cc

@@ -73,7 +73,7 @@ bool BuildLog::OpenForWrite(const string& path, string* err) {
 
 void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time,
                              TimeStamp restat_mtime) {
-  const string command = edge->EvaluateCommand();
+  string command = edge->EvaluateCommand(true);
   for (vector<Node*>::iterator out = edge->outputs_.begin();
        out != edge->outputs_.end(); ++out) {
     const string& path = (*out)->path();

src/build_test.cc

@@ -233,7 +233,9 @@ bool BuildTest::CanRunMore() {
 bool BuildTest::StartCommand(Edge* edge) {
   assert(!last_command_);
   commands_ran_.push_back(edge->EvaluateCommand());
-  if (edge->rule().name() == "cat" || edge->rule_->name() == "cc" ||
+  if (edge->rule().name() == "cat"  ||
+      edge->rule().name() == "cat_rsp" ||
+      edge->rule().name() == "cc" ||
       edge->rule().name() == "touch") {
     for (vector<Node*>::iterator out = edge->outputs_.begin();
          out != edge->outputs_.end(); ++out) {
@@ -808,3 +810,129 @@ TEST_F(BuildDryRun, AllCommandsShown) {
   ASSERT_EQ(3u, commands_ran_.size());
 }
 
+// Test that RSP files are created when & where appropriate and deleted after
+// succesful execution.
+TEST_F(BuildTest, RspFileSuccess)
+{
+  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+    "rule cat_rsp\n"
+    "  command = cat $rspfile > $out\n"
+    "  rspfile = $rspfile\n"
+    "  rspfile_content = $long_command\n"
+    "build out1: cat in\n"
+    "build out2: cat_rsp in\n"
+    "  rspfile = out2.rsp\n"
+    "  long_command = Some very long command\n"));
+
+  fs_.Create("out1", now_, "");
+  fs_.Create("out2", now_, "");
+  fs_.Create("out3", now_, "");
+
+  now_++;
+
+  fs_.Create("in", now_, "");
+
+  string err;
+  EXPECT_TRUE(builder_.AddTarget("out1", &err));
+  ASSERT_EQ("", err);
+  EXPECT_TRUE(builder_.AddTarget("out2", &err));
+  ASSERT_EQ("", err);
+
+  size_t files_created = fs_.files_created_.size();
+  size_t files_removed = fs_.files_removed_.size();
+
+  EXPECT_TRUE(builder_.Build(&err));
+  ASSERT_EQ(2, commands_ran_.size()); // cat + cat_rsp
+
+  // The RSP file was created
+  ASSERT_EQ(files_created + 1, fs_.files_created_.size());
+  ASSERT_EQ(1, fs_.files_created_.count("out2.rsp"));
+
+  // The RSP file was removed
+  ASSERT_EQ(files_removed + 1, fs_.files_removed_.size());
+  ASSERT_EQ(1, fs_.files_removed_.count("out2.rsp"));
+}
+
+// Test that RSP file is created but not removed for commands, which fail
+TEST_F(BuildTest, RspFileFailure) {
+  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+    "rule fail\n"
+    "  command = fail\n"
+    "  rspfile = $rspfile\n"
+    "  rspfile_content = $long_command\n"
+    "build out: fail in\n"
+    "  rspfile = out.rsp\n"
+    "  long_command = Another very long command\n"));
+
+  fs_.Create("out", now_, "");
+  now_++;
+  fs_.Create("in", now_, "");
+
+  string err;
+  EXPECT_TRUE(builder_.AddTarget("out", &err));
+  ASSERT_EQ("", err);
+
+  size_t files_created = fs_.files_created_.size();
+  size_t files_removed = fs_.files_removed_.size();
+
+  EXPECT_FALSE(builder_.Build(&err));
+  ASSERT_EQ("subcommand failed", err);
+  ASSERT_EQ(1, commands_ran_.size());
+
+  // The RSP file was created
+  ASSERT_EQ(files_created + 1, fs_.files_created_.size());
+  ASSERT_EQ(1, fs_.files_created_.count("out.rsp"));
+
+  // The RSP file was NOT removed
+  ASSERT_EQ(files_removed, fs_.files_removed_.size());
+  ASSERT_EQ(0, fs_.files_removed_.count("out.rsp"));
+
+  // The RSP file contains what it should
+  ASSERT_EQ("Another very long command", fs_.files_["out.rsp"].contents);
+}
+
+// Test that contens of the RSP file behaves like a regular part of 
+// command line, i.e. triggers a rebuild if changed
+TEST_F(BuildWithLogTest, RspFileCmdLineChange) {
+  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+    "rule cat_rsp\n"
+    "  command = cat $rspfile > $out\n"
+    "  rspfile = $rspfile\n"
+    "  rspfile_content = $long_command\n"
+    "build out: cat_rsp in\n"
+    "  rspfile = out.rsp\n"
+    "  long_command = Original very long command\n"));
+
+  fs_.Create("out", now_, "");
+  now_++;
+  fs_.Create("in", now_, "");
+
+  string err;
+  EXPECT_TRUE(builder_.AddTarget("out", &err));
+  ASSERT_EQ("", err);
+
+  // 1. Build for the 1st time (-> populate log)
+  EXPECT_TRUE(builder_.Build(&err));
+  ASSERT_EQ(1, commands_ran_.size());
+
+  // 2. Build again (no change)
+  commands_ran_.clear();
+  state_.Reset();
+  EXPECT_TRUE(builder_.AddTarget("out", &err));
+  EXPECT_EQ("", err);
+  ASSERT_TRUE(builder_.AlreadyUpToDate());
+
+  // 3. Alter the entry in the logfile 
+  // (to simulate a change in the command line between 2 builds)
+  BuildLog::LogEntry * log_entry = build_log_.LookupByOutput("out");
+  ASSERT_TRUE(NULL != log_entry);
+  ASSERT_EQ("cat out.rsp > out;rspfile=Original very long command", log_entry->command);
+  log_entry->command = "cat out.rsp > out;rspfile=Altered very long command";
+  // Now expect the target to be rebuilt
+  commands_ran_.clear();
+  state_.Reset();
+  EXPECT_TRUE(builder_.AddTarget("out", &err));
+  EXPECT_EQ("", err);
+  EXPECT_TRUE(builder_.Build(&err));
+  EXPECT_EQ(1, commands_ran_.size());
+}

src/clean.cc

@@ -106,13 +106,17 @@ int Cleaner::CleanAll(bool generator) {
       continue;
     // Do not remove generator's files unless generator specified.
     if (!generator && (*e)->rule().generator())
-      continue;
+      continue;    
     for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
          out_node != (*e)->outputs_.end(); ++out_node) {
       Remove((*out_node)->path());
     }
+    // Remove the depfile
     if (!(*e)->rule().depfile().empty())
       Remove((*e)->EvaluateDepFile());
+    // Remove the response file
+    if ((*e)->HasRspFile()) 
+      Remove((*e)->GetRspFile());      
   }
   PrintFooter();
   return status_;
@@ -121,6 +125,8 @@ int Cleaner::CleanAll(bool generator) {
 void Cleaner::DoCleanTarget(Node* target) {
   if (target->in_edge()) {
     Remove(target->path());
+    if (target->in_edge()->HasRspFile())
+      Remove(target->in_edge()->GetRspFile());
     for (vector<Node*>::iterator n = target->in_edge()->inputs_.begin();
          n != target->in_edge()->inputs_.end();
          ++n) {
@@ -181,6 +187,8 @@ void Cleaner::DoCleanRule(const Rule* rule) {
       for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
            out_node != (*e)->outputs_.end(); ++out_node) {
         Remove((*out_node)->path());
+        if ((*e)->HasRspFile()) 
+          Remove((*e)->GetRspFile());
       }
     }
   }

src/clean_test.cc

@@ -249,6 +249,67 @@ TEST_F(CleanTest, CleanDepFile) {
   EXPECT_EQ(2u, fs_.files_removed_.size());
 }
 
+TEST_F(CleanTest, CleanRspFile) {
+  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n"
+"  command = cc $in > $out\n"
+"  rspfile = $rspfile\n"
+"  rspfile_content=$in\n"
+"build out1: cc in1\n"
+"  rspfile = cc1.rsp\n"
+"  rspfile_content=$in\n"));
+  fs_.Create("out1", 1, "");
+  fs_.Create("cc1.rsp", 1, "");
+
+  Cleaner cleaner(&state_, config_, &fs_);
+  EXPECT_EQ(0, cleaner.CleanAll());
+  EXPECT_EQ(2, cleaner.cleaned_files_count());
+  EXPECT_EQ(2u, fs_.files_removed_.size());
+}
+
+TEST_F(CleanTest, CleanRsp) {
+  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cat_rsp \n"
+"  command = cat $rspfile > $out\n"
+"  rspfile = $rspfile\n"
+"  rspfile_content = $in\n"
+"build in1: cat src1\n"
+"build out1: cat in1\n"
+"build in2: cat_rsp src2\n"
+"  rspfile=in2.rsp\n"
+"  rspfile_content=$in\n"
+"build out2: cat_rsp in2\n"
+"  rspfile=out2.rsp\n"
+"  rspfile_content=$in\n"));
+  fs_.Create("in1", 1, "");
+  fs_.Create("out1", 1, "");
+  fs_.Create("in2.rsp", 1, "");
+  fs_.Create("out2.rsp", 1, "");
+  fs_.Create("in2", 1, "");
+  fs_.Create("out2", 1, "");
+
+  Cleaner cleaner(&state_, config_, &fs_);
+  ASSERT_EQ(0, cleaner.cleaned_files_count());
+  ASSERT_EQ(0, cleaner.CleanTarget("out1"));
+  EXPECT_EQ(2, cleaner.cleaned_files_count()); 
+  ASSERT_EQ(0, cleaner.CleanTarget("in2"));
+  EXPECT_EQ(2, cleaner.cleaned_files_count());
+  ASSERT_EQ(0, cleaner.CleanRule("cat_rsp"));
+  EXPECT_EQ(2, cleaner.cleaned_files_count());
+
+  EXPECT_EQ(6u, fs_.files_removed_.size());
+
+  // Check they are removed.
+  EXPECT_EQ(0, fs_.Stat("in1"));
+  EXPECT_EQ(0, fs_.Stat("out1"));
+  EXPECT_EQ(0, fs_.Stat("in2"));
+  EXPECT_EQ(0, fs_.Stat("out2"));
+  EXPECT_EQ(0, fs_.Stat("in2.rsp"));
+  EXPECT_EQ(0, fs_.Stat("out2.rsp"));
+
+  fs_.files_removed_.clear();
+}
+
 TEST_F(CleanTest, CleanFailure) {
   ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
                                       "build dir: cat src1\n"));

src/disk_interface.cc

@@ -103,6 +103,26 @@ TimeStamp RealDiskInterface::Stat(const string& path) {
 #endif
 }
 
+bool RealDiskInterface::WriteFile(const string & path, const string & contents) {
+  FILE * fp = fopen(path.c_str(), "w");
+  if (fp == NULL) {
+    Error("WriteFile(%s): Unable to create file. %s", path.c_str(), strerror(errno));
+    return false;
+  }
+
+  if (fwrite(contents.data(), 1, contents.length(), fp) < contents.length())  {
+    Error("WriteFile(%s): Unable to write to the file. %s", path.c_str(), strerror(errno));
+    return false;
+  }
+
+  if (fclose(fp) == EOF) {
+    Error("WriteFile(%s): Unable to close the file. %s", path.c_str(), strerror(errno));
+    return false;
+  }
+
+  return true;
+}
+
 bool RealDiskInterface::MakeDir(const string& path) {
   if (::MakeDir(path) < 0) {
     Error("mkdir(%s): %s", path.c_str(), strerror(errno));

src/disk_interface.h

@@ -34,6 +34,10 @@ struct DiskInterface {
   /// Create a directory, returning false on failure.
   virtual bool MakeDir(const string& path) = 0;
 
+  /// Create a file, with the specified name and contens
+  /// Returns true on success, false on failure
+  virtual bool WriteFile(const string & path, const string & contetns) = 0;
+
   /// Read a file to a string.  Fill in |err| on error.
   virtual string ReadFile(const string& path, string* err) = 0;
 
@@ -54,6 +58,7 @@ struct RealDiskInterface : public DiskInterface {
   virtual ~RealDiskInterface() {}
   virtual TimeStamp Stat(const string& path);
   virtual bool MakeDir(const string& path);
+  virtual bool WriteFile(const string & path, const string & contens);
   virtual string ReadFile(const string& path, string* err);
   virtual int RemoveFile(const string& path);
 };

src/disk_interface_test.cc

@@ -102,6 +102,10 @@ struct StatTest : public StateTestWithBuiltinRules,
                   public DiskInterface {
   // DiskInterface implementation.
   virtual TimeStamp Stat(const string& path);
+  virtual bool WriteFile(const string& path, const string & contents) {
+      assert(false);
+      return true;
+  }
   virtual bool MakeDir(const string& path) {
     assert(false);
     return false;