[C++] Re-generate ninja file when a file is added/removed

With this change, we store the results of file list related
commands in .kati_stamp. If one of them has been changed,
we re-generate ninja file.

Currently, this check is slow. We need to check the timestamp
of directories first like what we are doing for $(wildcard).

exec.cc

@@ -104,7 +104,8 @@ class Executor {
       }
       if (!g_is_dry_run) {
         string out;
-        int result = RunCommand(*shell_, command->cmd->c_str(), true,
+        int result = RunCommand(*shell_, command->cmd->c_str(),
+                                RedirectStderr::STDOUT,
                                 &out);
         printf("%s", out.c_str());
         if (result != 0) {

fileutil.cc

@@ -17,6 +17,7 @@
 #include "fileutil.h"
 
 #include <errno.h>
+#include <fcntl.h>
 #include <glob.h>
 #include <limits.h>
 #include <sys/stat.h>
@@ -53,7 +54,8 @@ double GetTimestamp(StringPiece filename) {
 #endif
 }
 
-int RunCommand(const string& shell, const string& cmd, bool redirect_stderr,
+int RunCommand(const string& shell, const string& cmd,
+               RedirectStderr redirect_stderr,
                string* s) {
   int pipefd[2];
   if (pipe(pipefd) != 0)
@@ -86,9 +88,14 @@ int RunCommand(const string& shell, const string& cmd, bool redirect_stderr,
     return status;
   } else {
     close(pipefd[0]);
-    if (redirect_stderr) {
+    if (redirect_stderr == RedirectStderr::STDOUT) {
       if (dup2(pipefd[1], 2) < 0)
         PERROR("dup2 failed");
+    } else if (redirect_stderr == RedirectStderr::DEV_NULL) {
+      int fd = open("/dev/null", O_WRONLY);
+      if (dup2(fd, 2) < 0)
+        PERROR("dup2 failed");
+      close(fd);
     }
     if (dup2(pipefd[1], 1) < 0)
       PERROR("dup2 failed");

fileutil.h

@@ -27,7 +27,14 @@ using namespace std;
 bool Exists(StringPiece f);
 double GetTimestamp(StringPiece f);
 
-int RunCommand(const string& shell, const string& cmd, bool redirect_stderr,
+enum struct RedirectStderr {
+  NONE,
+  STDOUT,
+  DEV_NULL,
+};
+
+int RunCommand(const string& shell, const string& cmd,
+               RedirectStderr redirect_stderr,
                string* out);
 
 void GetExecutablePath(string* path);

find.cc

@@ -574,11 +574,8 @@ class FindEmulatorImpl : public FindEmulator {
     return r;
   }
 
-  virtual bool HandleFind(const string& cmd, string* out) override {
-    FindCommand fc;
-    if (!fc.Parse(cmd))
-      return false;
-
+  virtual bool HandleFind(const string& cmd UNUSED, const FindCommand& fc,
+                          string* out) override {
     if (HasPrefix(fc.chdir, "/")) {
       LOG("FindEmulator: Cannot handle abspath: %s", cmd.c_str());
       return false;
@@ -676,12 +673,13 @@ FindCommand::FindCommand()
     : follows_symlinks(false), depth(INT_MAX) {
 }
 
+FindCommand::~FindCommand() {
+}
+
 bool FindCommand::Parse(const string& cmd) {
   FindCommandParser fcp(cmd, this);
-  size_t found = cmd.find("find ");
-  if (found == string::npos || (found != 0 && !isspace(cmd[found-1]))) {
+  if (!HasWord(cmd, "find"))
     return false;
-  }
 
   if (!fcp.Parse())
     return false;

find.h

@@ -27,7 +27,7 @@ class FindCond;
 
 struct FindCommand {
   FindCommand();
-  ~FindCommand() = default;
+  ~FindCommand();
 
   bool Parse(const string& cmd);
 
@@ -44,7 +44,8 @@ class FindEmulator {
  public:
   virtual ~FindEmulator() = default;
 
-  virtual bool HandleFind(const string& cmd, string* out) = 0;
+  virtual bool HandleFind(const string& cmd, const FindCommand& fc,
+                          string* out) = 0;
 
   static FindEmulator* Get();
 

func.cc

@@ -462,18 +462,6 @@ void EvalFunc(const vector<Value*>& args, Evaluator* ev, string*) {
 
 //#define TEST_FIND_EMULATOR
 
-#ifdef TEST_FIND_EMULATOR
-static string SortWordsInString(StringPiece s) {
-  vector<string> toks;
-  for (StringPiece tok : WordScanner(s)) {
-    toks.push_back(tok.as_string());
-  }
-  sort(toks.begin(), toks.end());
-  return JoinStrings(toks, " ");
-}
-#endif
-
-
 // A hack for Android build. We need to evaluate things like $((3+4))
 // when we emit ninja file, because the result of such expressions
 // will be passed to other make functions.
@@ -487,51 +475,81 @@ static bool HasNoIoInShellScript(const string& cmd) {
   return false;
 }
 
-void ShellFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
-  shared_ptr<string> cmd = args[0]->Eval(ev);
-  if (ev->avoid_io() && !HasNoIoInShellScript(*cmd)) {
-    *s += "$(";
-    *s += *StripShellComment(cmd);
-    *s += ")";
-    return;
-  }
-
-  LOG("ShellFunc: %s", cmd->c_str());
+static void ShellFuncImpl(const string& shell, const string& cmd,
+                          bool is_file_list_command,
+                          string* s, FindCommand** fc) {
+  LOG("ShellFunc: %s", cmd.c_str());
 
 #ifdef TEST_FIND_EMULATOR
   bool need_check = false;
   string out2;
-  if (FindEmulator::Get() && FindEmulator::Get()->HandleFind(*cmd, &out2)) {
-    need_check = true;
-  }
+#endif
+  if (FindEmulator::Get()) {
+    *fc = new FindCommand();
+    if ((*fc)->Parse(cmd)) {
+#ifdef TEST_FIND_EMULATOR
+      if (FindEmulator::Get()->HandleFind(cmd, **fc, &out2)) {
+        need_check = true;
+      }
 #else
-  if (FindEmulator::Get() && FindEmulator::Get()->HandleFind(*cmd, s))
-    return;
+      if (FindEmulator::Get()->HandleFind(cmd, **fc, s)) {
+        *s = SortWordsInString(*s);
+        return;
+      }
 #endif
+    }
+    delete *fc;
+    *fc = NULL;
+  }
 
-  COLLECT_STATS_WITH_SLOW_REPORT("func shell time", cmd->c_str());
-  string out;
-  shared_ptr<string> shell = ev->EvalVar(kShellSym);
-  RunCommand(*shell, *cmd, false, &out);
-
-  while (out[out.size()-1] == '\n')
-    out.pop_back();
-  for (size_t i = 0; i < out.size(); i++) {
-    if (out[i] == '\n')
-      out[i] = ' ';
+  COLLECT_STATS_WITH_SLOW_REPORT("func shell time", cmd.c_str());
+  RunCommand(shell, cmd, RedirectStderr::NONE, s);
+  if (is_file_list_command) {
+    *s = SortWordsInString(*s);
+  } else {
+    FormatForCommandSubstitution(s);
   }
 
 #ifdef TEST_FIND_EMULATOR
   if (need_check) {
-    string sorted = SortWordsInString(out);
+    string sorted = SortWordsInString(*s);
     out2 = SortWordsInString(out2);
     if (sorted != out2) {
       ERROR("FindEmulator is broken: %s\n%s\nvs\n%s",
-            cmd->c_str(), sorted.c_str(), out2.c_str());
+            cmd.c_str(), sorted.c_str(), out2.c_str());
     }
   }
 #endif
+}
+
+static vector<FileListCommand*> g_file_list_commands;
+
+void ShellFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
+  shared_ptr<string> cmd = args[0]->Eval(ev);
+  if (ev->avoid_io() && !HasNoIoInShellScript(*cmd)) {
+    *s += "$(";
+    *s += *StripShellComment(cmd);
+    *s += ")";
+    return;
+  }
 
+  shared_ptr<string> shell = ev->EvalVar(kShellSym);
+  const bool is_file_list_command =
+      ((*shell == "/bin/sh" || *shell == "/bin/bash") &&
+       (HasWord(*cmd, "find") || HasWord(*cmd, "ls") ||
+        // For Android.
+        cmd->find("/findleaves.py ") != string::npos));
+
+  string out;
+  FindCommand* fc = NULL;
+  ShellFuncImpl(*shell, *cmd, is_file_list_command, &out, &fc);
+  if (is_file_list_command) {
+    FileListCommand* flc = new FileListCommand();
+    flc->cmd = *cmd;
+    flc->find.reset(fc);
+    flc->result = out;
+    g_file_list_commands.push_back(flc);
+  }
   *s += out;
 }
 
@@ -688,3 +706,7 @@ FuncInfo* GetFuncInfo(StringPiece name) {
     return NULL;
   return found->second;
 }
+
+const vector<FileListCommand*>& GetFileListCommmands() {
+  return g_file_list_commands;
+}

func.h

@@ -15,6 +15,8 @@
 #ifndef FUNC_H_
 #define FUNC_H_
 
+#include <memory>
+#include <string>
 #include <vector>
 
 #include "value.h"
@@ -37,4 +39,14 @@ void QuitFuncTable();
 
 FuncInfo* GetFuncInfo(StringPiece name);
 
+struct FindCommand;
+
+struct FileListCommand {
+  string cmd;
+  unique_ptr<FindCommand> find;
+  string result;
+};
+
+const vector<FileListCommand*>& GetFileListCommmands();
+
 #endif  // FUNC_H_

ninja.cc

@@ -34,6 +34,7 @@
 #include "file_cache.h"
 #include "fileutil.h"
 #include "flags.h"
+#include "func.h"
 #include "io.h"
 #include "log.h"
 #include "string_piece.h"
@@ -750,6 +751,13 @@ class NinjaGenerator {
       }
     }
 
+    const vector<FileListCommand*>& flcs = GetFileListCommmands();
+    DumpInt(fp, flcs.size());
+    for (FileListCommand* flc : flcs) {
+      DumpString(fp, flc->cmd);
+      DumpString(fp, flc->result);
+    }
+
     fclose(fp);
   }
 
@@ -862,5 +870,21 @@ bool NeedsRegen(const char* ninja_suffix,
     }
   }
 
+  int num_flcs = LoadInt(fp);
+  for (int i = 0; i < num_flcs; i++) {
+    string cmd;
+    LoadString(fp, &cmd);
+    string result;
+    RunCommand("/bin/sh", cmd, RedirectStderr::DEV_NULL, &result);
+    FormatForCommandSubstitution(&result);
+    result = SortWordsInString(result);
+    LoadString(fp, &s);
+    if (s != result) {
+      fprintf(stderr, "$(shell %s) was changed, regenerating...\n",
+              cmd.c_str());
+      return true;
+    }
+  }
+
   return false;
 }

strutil.cc

@@ -20,6 +20,7 @@
 #include <limits.h>
 #include <unistd.h>
 
+#include <algorithm>
 #include <stack>
 #include <utility>
 
@@ -115,6 +116,18 @@ bool HasSuffix(StringPiece str, StringPiece suffix) {
   return size_diff >= 0 && str.substr(size_diff) == suffix;
 }
 
+bool HasWord(StringPiece str, StringPiece w) {
+  size_t found = str.find(w);
+  if (found == string::npos)
+    return false;
+  if (found != 0 && !isspace(str[found-1]))
+    return false;
+  size_t end = found + w.size();
+  if (end != str.size() && !isspace(str[end]))
+    return false;
+  return true;
+}
+
 StringPiece TrimSuffix(StringPiece str, StringPiece suffix) {
   ssize_t size_diff = str.size() - suffix.size();
   if (size_diff < 0 || str.substr(size_diff) != suffix)
@@ -375,3 +388,21 @@ StringPiece TrimLeadingCurdir(StringPiece s) {
     s = s.substr(2);
   return s;
 }
+
+void FormatForCommandSubstitution(string* s) {
+  while ((*s)[s->size()-1] == '\n')
+    s->pop_back();
+  for (size_t i = 0; i < s->size(); i++) {
+    if ((*s)[i] == '\n')
+      (*s)[i] = ' ';
+  }
+}
+
+string SortWordsInString(StringPiece s) {
+  vector<string> toks;
+  for (StringPiece tok : WordScanner(s)) {
+    toks.push_back(tok.as_string());
+  }
+  sort(toks.begin(), toks.end());
+  return JoinStrings(toks, " ");
+}

strutil.h

@@ -87,6 +87,8 @@ bool HasPrefix(StringPiece str, StringPiece prefix);
 
 bool HasSuffix(StringPiece str, StringPiece suffix);
 
+bool HasWord(StringPiece str, StringPiece w);
+
 StringPiece TrimSuffix(StringPiece str, StringPiece suffix);
 
 class Pattern {
@@ -130,4 +132,8 @@ size_t FindEndOfLine(StringPiece s, size_t e, size_t* lf_cnt);
 // From http://www.gnu.org/software/make/manual/make.html#Features
 StringPiece TrimLeadingCurdir(StringPiece s);
 
+void FormatForCommandSubstitution(string* s);
+
+string SortWordsInString(StringPiece s);
+
 #endif  // STRUTIL_H_