Commit count: Listing ignored files could hang GE

Issue gitextensions#4256
Listing ignored files with a separate "ls-files -o -i --exclude-standard" seem to never exit in some situations, see gitextensions#4256
This reuses the git-status call with --ignored as an option to get the ignored files. (The option is only added when ls-files were called previously.) This call did not seem to hang with many ignored files and the command runtime does not increase much.

The solution is not optional though. It still lists all ignored files in memory. The time to find ignored files is faster, but a lot of memory is needed. gitextensions#4256 has some discussions.
Also, processing all output to GitStatusItem first before creating the HashSet will require some more memory (temporary) than required.

GitUI/CommandsDialogs/BrowseDialog/GitStatusMonitor.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using GitCommands;
 using GitUIPluginInterfaces;
 
@@ -26,6 +27,8 @@ public partial class GitStatusMonitor : IDisposable
 
         private bool _commandIsRunning;
         private bool _statusIsUpToDate = true;
+        private bool _ignoredFilesPending;
+
         private readonly FileSystemWatcher _workTreeWatcher = new FileSystemWatcher();
         private readonly FileSystemWatcher _gitDirWatcher = new FileSystemWatcher();
         private readonly FileSystemWatcher _globalIgnoreWatcher = new FileSystemWatcher();
@@ -128,6 +131,8 @@ private GitStatusMonitorState CurrentStatus
                             _globalIgnoreWatcher.EnableRaisingEvents = !string.IsNullOrWhiteSpace(_globalIgnoreWatcher.Path);
                             _currUpdateInterval = MinUpdateInterval;
                             _prevUpdateTime = 0;
+                            _ignoredFilesAreStale = true;
+                            _ignoredFiles = new HashSet<string>();
                             ScheduleNext(UpdateDelay);
                         }
                         break;
@@ -219,7 +224,6 @@ private void StartWatchingChanges(string workTreePath, string gitDirPath)
                     }
                     _gitPath = Path.GetDirectoryName(gitDirPath);
                     _submodulesPath = Path.Combine(_gitPath, "modules");
-                    UpdateIgnoredFiles(true);
                     ignoredFilesTimer.Stop();
                     ignoredFilesTimer.Start();
                     CurrentStatus = GitStatusMonitorState.Running;
@@ -235,29 +239,6 @@ private void StartWatchingChanges(string workTreePath, string gitDirPath)
             }
         }
 
-        private HashSet<string> LoadIgnoredFiles()
-        {
-            string lsOutput = Module.RunGitCmd("ls-files -o -i --exclude-standard");
-            string[] tab = lsOutput.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries);
-            return new HashSet<string>(tab);
-        }
-
-        private void UpdateIgnoredFiles(bool clearImmediately)
-        {
-            if (clearImmediately)
-                _ignoredFiles = new HashSet<string>();
-
-            AsyncLoader.DoAsync(
-                LoadIgnoredFiles,
-                (ignoredSet) =>
-                {
-                    _ignoredFiles = ignoredSet;
-                    _ignoredFilesAreStale = false;
-                },
-                (e) => { _ignoredFiles = new HashSet<string>(); }
-            );
-        }
-
         private void Update()
         {
             if (CurrentStatus != GitStatusMonitorState.Running)
@@ -283,10 +264,6 @@ private void Update()
                 _commandIsRunning = true;
                 _statusIsUpToDate = true;
                 _prevUpdateTime = Environment.TickCount;
-                if (_ignoredFilesAreStale)
-                {
-                    UpdateIgnoredFiles(false);
-                }
                 AsyncLoader.DoAsync(RunStatusCommand, UpdatedStatusReceived, OnUpdateStatusError);
                 // Schedule update every 5 min, even if we don't know that anything changed
                 ScheduleNext(MaxUpdatePeriod);
@@ -295,7 +272,9 @@ private void Update()
 
         private string RunStatusCommand()
         {
-            string command = GitCommandHelpers.GetAllChangedFilesCmd(true, UntrackedFilesMode.Default);
+            _ignoredFilesPending = _ignoredFilesAreStale;
+            //Note: Git 2.15 introduces --ignored=matching to only get ignored directories, decreasing size
+            string command = GitCommandHelpers.GetAllChangedFilesCmd(!_ignoredFilesPending, UntrackedFilesMode.Default);
             return Module.RunGitCmd(command);
         }
 
@@ -316,6 +295,15 @@ private void UpdatedStatusReceived(string updatedStatus)
 
             var allChangedFiles = GitCommandHelpers.GetAllChangedFilesFromString(Module, updatedStatus);
             OnGitWorkingDirectoryStatusChanged(new GitWorkingDirectoryStatusEventArgs(allChangedFiles));
+            if (_ignoredFilesPending)
+            {
+                _ignoredFilesPending = false;
+                _ignoredFiles = new HashSet<string>(allChangedFiles.Where(item => item.IsIgnored).Select(item => item.Name).ToArray());
+                if (_statusIsUpToDate)
+                {
+                    _ignoredFilesAreStale = false;
+                }
+            }
             if (!_statusIsUpToDate)
             {
                 //Still not up-to-date, but present what received, GetAllChangedFilesCmd() is the heavy command
@@ -395,12 +383,13 @@ private void GitUICommands_PostCheckout(object sender, GitUIPostActionEventArgs
 
         private void GitUICommands_PostEditGitIgnore(object sender, GitUIBaseEventArgs e)
         {
-            UpdateIgnoredFiles(true);
+            _ignoredFiles = new HashSet<string>();
+            _ignoredFilesAreStale = true;
         }
 
         private void ignoredFilesTimer_Tick(object sender, EventArgs e)
         {
-            UpdateIgnoredFiles(false);
+            _ignoredFilesAreStale = true;
         }
 
         private void timerRefresh_Tick(object sender, EventArgs e)
@@ -420,8 +409,6 @@ private void WorkTreeChanged(object sender, FileSystemEventArgs e)
             if (_nextUpdateTime < Environment.TickCount + UpdateDelay)
                 return;
 
-            //TODO Smarter detection of ignored files created since the files were created
-            //Parse .gitignore?
             var fileName = e.FullPath.Substring(_workTreeWatcher.Path.Length).ToPosixPath();
             if (_ignoredFiles.Contains(fileName))
                 return;