Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Modify IdleManager to use CPU as a measure to identify if a process is

idling

Fixes #485
  • Loading branch information...
commit a068c42f08d4d6a52c070181dad3d6391404d7d8 1 parent 64db3d8
@pranavkm pranavkm authored
View
8 Common/Constants.cs
@@ -1,3 +1,4 @@
+using System;
namespace Kudu
{
public static class Constants
@@ -32,5 +33,12 @@ public static class Constants
public const string DiagnosticsPath = @"diagnostics";
public const string SettingsJsonFile = @"settings.json";
+
+ private static readonly TimeSpan _maxAllowedExectionTime = TimeSpan.FromMinutes(30);
+
+ public static TimeSpan MaxAllowedExecutionTime
+ {
+ get { return _maxAllowedExectionTime; }
+ }
}
}
View
2  Kudu.Contracts/Settings/DeploymentSettingsExtension.cs
@@ -7,7 +7,7 @@ namespace Kudu.Contracts.Settings
{
public static class DeploymentSettingsExtension
{
- public static readonly TimeSpan DefaultCommandIdleTimeout = TimeSpan.FromMinutes(10);
+ public static readonly TimeSpan DefaultCommandIdleTimeout = TimeSpan.FromMinutes(6);
public static readonly TimeSpan DefaultLogStreamTimeout = TimeSpan.FromMinutes(30);
public const TraceLevel DefaultTraceLevel = TraceLevel.Error;
View
191 Kudu.Core.Test/IdleManagerFacts.cs
@@ -0,0 +1,191 @@
+using System;
+using Kudu.Contracts.Tracing;
+using Kudu.Core.Infrastructure;
+using Moq;
+using Xunit;
+
+namespace Kudu.Core.Test
+{
+ public class IdleManagerFacts
+ {
+ [Fact]
+ public void WaitForExitWaitsForEOFPriorToExiting()
+ {
+ // Arrange
+ var process = new Mock<IProcess>(MockBehavior.Strict);
+ process.Setup(f => f.WaitForExit(It.IsAny<TimeSpan>()))
+ .Returns(true)
+ .Verifiable();
+ process.Setup(f => f.WaitUntilEOF())
+ .Verifiable();
+ var idleManager = new IdleManager(TimeSpan.FromSeconds(5), Mock.Of<ITracer>());
+
+ // Act
+ idleManager.WaitForExit(process.Object);
+
+ // Assert
+ process.Verify();
+ }
+
+ [Fact]
+ public void WaitForExitPollsAllowsExecutableToContinueAfterTimeoutIfItIsBusy()
+ {
+ // Arrange
+ TimeSpan idleTimeout = TimeSpan.FromMinutes(10);
+ IdleManager idleManager = new IdleManager(idleTimeout, Mock.Of<ITracer>());
+ var process = new Mock<IProcess>(MockBehavior.Strict);
+ process.Setup(f => f.WaitForExit(idleTimeout))
+ .Returns(false)
+ .Verifiable();
+ int num = 0;
+ process.Setup(f => f.WaitForExit(TimeSpan.FromSeconds(10)))
+ .Returns(() =>
+ {
+ if (num++ == 3)
+ {
+ return true;
+ }
+ else
+ {
+ idleManager.UpdateActivity();
+ return false;
+ }
+ });
+ process.Setup(f => f.GetTotalProcessorTime())
+ .Returns(num);
+ process.Setup(f => f.WaitUntilEOF())
+ .Verifiable();
+
+ // Act
+ idleManager.WaitForExit(process.Object);
+
+ // Assert
+ process.Verify();
+ }
+
+ [Fact]
+ public void WaitForExitPollsAllowsExecutableToContinueAsLongAsItIsPerformingSomeCPUOrUpdating()
+ {
+ // Arrange
+ TimeSpan idleTimeout = TimeSpan.FromMinutes(10);
+ IdleManager idleManager = new IdleManager(idleTimeout, Mock.Of<ITracer>());
+ var process = new Mock<IProcess>(MockBehavior.Strict);
+ process.Setup(f => f.WaitForExit(idleTimeout))
+ .Returns(false);
+ int num = 0, cpu = 0;
+ process.Setup(f => f.WaitForExit(TimeSpan.FromSeconds(10)))
+ .Returns(() =>
+ {
+ if (num++ == 10)
+ {
+ return true;
+ }
+ else if (num % 2 == 0)
+ {
+ idleManager.UpdateActivity();
+ }
+ else
+ {
+ cpu++;
+ }
+ return false;
+ });
+ process.Setup(f => f.GetTotalProcessorTime())
+ .Returns(cpu);
+ process.Setup(f => f.WaitUntilEOF())
+ .Verifiable();
+
+ // Act
+ idleManager.WaitForExit(process.Object);
+
+ // Assert
+ process.Verify();
+ }
+
+ [Fact]
+ public void WaitForExitPollsKillsProcessIfProcessorTimeDoesNotChangeAndNotUpdated()
+ {
+ // Arrange
+ var tracer = Mock.Of<ITracer>();
+ var idleTimeout = TimeSpan.FromMinutes(10);
+ IdleManager idleManager = new IdleManager(idleTimeout, tracer, DateTime.UtcNow.AddMinutes(-1));
+ var process = new Mock<IProcess>(MockBehavior.Strict);
+ process.SetupGet(f => f.Name)
+ .Returns("Test-Process");
+ process.SetupGet(f => f.Arguments)
+ .Returns("");
+ process.Setup(f => f.WaitForExit(idleTimeout))
+ .Returns(false);
+ process.Setup(f => f.WaitForExit(TimeSpan.FromSeconds(10)))
+ .Returns(false);
+ process.Setup(f => f.GetTotalProcessorTime())
+ .Returns(5);
+ process.Setup(f => f.Kill(tracer))
+ .Verifiable();
+
+ // Act
+ var ex = Assert.Throws<CommandLineException>(() => idleManager.WaitForExit(process.Object));
+
+ // Assert
+ process.Verify();
+
+ Assert.Contains("Command 'Test-Process ' aborted due to idle timeout after", ex.Message);
+ }
+
+ [Fact]
+ public void WaitForExitPollsKillsProcessIfItUpdatesActivityForOver30Minutes()
+ {
+ // Arrange
+ var tracer = Mock.Of<ITracer>();
+ IdleManager idleManager = new IdleManager(TimeSpan.FromMinutes(10), tracer);
+ var process = new Mock<IProcess>(MockBehavior.Strict);
+ process.SetupGet(f => f.Name)
+ .Returns("Test-Process");
+ process.SetupGet(f => f.Arguments)
+ .Returns("");
+ process.Setup(f => f.WaitForExit(It.IsAny<TimeSpan>()))
+ .Callback(() => idleManager.UpdateActivity())
+ .Returns(false);
+ process.Setup(f => f.GetTotalProcessorTime())
+ .Returns(5);
+ process.Setup(f => f.Kill(tracer))
+ .Verifiable();
+
+ // Act
+ var ex = Assert.Throws<CommandLineException>(() => idleManager.WaitForExit(process.Object));
+
+ // Assert
+ process.Verify();
+
+ Assert.Equal("Command 'Test-Process ' aborted due to idle timeout after '1800' seconds.\r\nTest-Process", ex.Message.TrimEnd());
+ }
+
+ [Fact]
+ public void WaitForExitPollsKillsProcessIfItIsCosntantlyUsingCPUForOver30Minutes()
+ {
+ // Arrange
+ var tracer = Mock.Of<ITracer>();
+ IdleManager idleManager = new IdleManager(TimeSpan.FromMinutes(10), tracer);
+ var process = new Mock<IProcess>(MockBehavior.Strict);
+ long i = 0;
+ process.SetupGet(f => f.Name)
+ .Returns("Test-Process");
+ process.SetupGet(f => f.Arguments)
+ .Returns("");
+ process.Setup(f => f.WaitForExit(It.IsAny<TimeSpan>()))
+ .Returns(false);
+ process.Setup(f => f.GetTotalProcessorTime())
+ .Returns(() => ++i);
+ process.Setup(f => f.Kill(tracer))
+ .Verifiable();
+
+ // Act
+ var ex = Assert.Throws<CommandLineException>(() => idleManager.WaitForExit(process.Object));
+
+ // Assert
+ process.Verify();
+
+ Assert.Equal("Command 'Test-Process ' aborted due to idle timeout after '1800' seconds.\r\nTest-Process", ex.Message.TrimEnd());
+ }
+ }
+}
View
1  Kudu.Core.Test/Kudu.Core.Test.csproj
@@ -55,6 +55,7 @@
<Compile Include="Deployment\DeploymentManagerFacts.cs" />
<Compile Include="Deployment\WapBuilderFacts.cs" />
<Compile Include="HgRepositoryFacts.cs" />
+ <Compile Include="IdleManagerFacts.cs" />
<Compile Include="Infrastructure\MsBuildSiteBuilderFacts.cs" />
<Compile Include="Infrastructure\DisposableActionFacts.cs" />
<Compile Include="Infrastructure\ExecutableExtensionFacts.cs" />
View
75 Kudu.Core/Infrastructure/Executable.cs
@@ -84,9 +84,9 @@ public void SetHomePath(string homePath)
process.Start();
#if !SITEMANAGEMENT
- var idleManager = new IdleManager(Path, IdleTimeout, tracer);
+ var idleManager = new Kudu.Core.Infrastructure.IdleManager(IdleTimeout, tracer);
#else
- var idleManager = new IdleManager();
+ var idleManager = new Kudu.SiteManagement.IdleManager();
#endif
Func<StreamReader, string> reader = (StreamReader streamReader) =>
{
@@ -150,7 +150,7 @@ public void Execute(ITracer tracer, Stream input, Stream output, string argument
var process = CreateProcess(arguments, args);
process.Start();
- var idleManager = new IdleManager(Path, IdleTimeout, tracer);
+ var idleManager = new IdleManager(IdleTimeout, tracer);
Func<StreamReader, string> reader = (StreamReader streamReader) => streamReader.ReadToEnd();
Action<Stream, Stream, bool> copyStream = (Stream from, Stream to, bool closeAfterCopy) =>
{
@@ -306,7 +306,7 @@ public void Execute(ITracer tracer, Stream input, Stream output, string argument
var errorBuffer = new StringBuilder();
var outputBuffer = new StringBuilder();
- var idleManager = new IdleManager(Path, IdleTimeout, tracer);
+ var idleManager = new IdleManager(IdleTimeout, tracer);
process.OutputDataReceived += (sender, e) =>
{
idleManager.UpdateActivity();
@@ -417,72 +417,5 @@ internal Process CreateProcess(string arguments, object[] args)
return process;
}
-
-#if !SITEMANAGEMENT
- class IdleManager
- {
- private static int WaitInterval = 5000;
- private readonly string _processName;
- private readonly TimeSpan _idleTimeout;
- private readonly ITracer _tracer;
- private DateTime _lastActivity;
-
- public IdleManager(string path, TimeSpan idleTimeout, ITracer tracer)
- {
- _processName = new FileInfo(path).Name;
- _idleTimeout = idleTimeout;
- _tracer = tracer;
- _lastActivity = DateTime.UtcNow;
- }
-
- public void UpdateActivity()
- {
- _lastActivity = DateTime.UtcNow;
- }
-
- public void WaitForExit(Process process)
- {
- while (!process.WaitForExit(WaitInterval))
- {
- if (DateTime.UtcNow > _lastActivity.Add(_idleTimeout))
- {
- process.Kill(true, _tracer);
- string message = String.Format(Resources.Error_ProcessAborted, _processName);
- throw new CommandLineException(process.StartInfo.FileName, process.StartInfo.Arguments, message)
- {
- ExitCode = -1,
- Output = message,
- Error = message
- };
- }
- }
-
- // Once we are here, the process has terminated. This extra WaitForExit with -1 timeout
- // will ensure in-memory Output buffer is flushed, from reflection, this.output.WaitUtilEOF().
- // If we don't do this, the leftover output will write concurrently to the logger
- // with the main thread corrupting the log xml.
- process.WaitForExit(-1);
- }
- }
-#else
- class IdleManager
- {
- public IdleManager()
- {
- }
-
- [SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic", Justification = "By design")]
- public void UpdateActivity()
- {
- }
-
- [SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic", Justification = "By design")]
- [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode", Justification = "Method is used, misdiagnosed due to linking of this file")]
- public void WaitForExit(Process process)
- {
- process.WaitForExit();
- }
- }
-#endif
}
}
View
47 Kudu.Core/Infrastructure/IProcess.cs
@@ -0,0 +1,47 @@
+using System;
+using Kudu.Contracts.Tracing;
+
+namespace Kudu.Core.Infrastructure
+{
+ public interface IProcess
+ {
+ /// <summary>
+ /// The file name of the executable
+ /// </summary>
+ string Name { get; }
+
+ /// <summary>
+ /// The arguments passed in to the executable.
+ /// </summary>
+ string Arguments { get; }
+
+ /// <summary>
+ /// Equivalent to Process.WaitForExit(-1)
+ /// </summary>
+ /// <remarks>
+ /// Once we are here, the process has terminated. This extra WaitForExit with -1 timeout
+ /// will ensure in-memory Output buffer is flushed, from reflection, this.output.WaitUtilEOF().
+ /// If we don't do this, the leftover output will write concurrently to the logger
+ /// with the main thread corrupting the log xml.
+ /// </remarks>
+ void WaitUntilEOF();
+
+ /// <summary>
+ /// Waits for the at most specified duration for the process to exit.
+ /// </summary>
+ /// <param name="timeSpan">The maximum duration to wait for.</param>
+ /// <returns></returns>
+ bool WaitForExit(TimeSpan timeSpan);
+
+ /// <summary>
+ /// Kills the process and all child processes spawned by it.
+ /// </summary>
+ void Kill(ITracer tracer);
+
+ /// <summary>
+ /// Gets the TotalProcessTime for the process tree in milliseconds
+ /// </summary>
+ /// <see cref="http://msdn.microsoft.com/en-us/library/system.diagnostics.process.totalprocessortime.aspx"/>
+ long GetTotalProcessorTime();
+ }
+}
View
95 Kudu.Core/Infrastructure/IdleManager.cs
@@ -0,0 +1,95 @@
+using System;
+using System.Diagnostics;
+using Kudu.Contracts.Tracing;
+
+namespace Kudu.Core.Infrastructure
+{
+ internal class IdleManager
+ {
+ private static readonly TimeSpan WaitIntervalTimeSpan = TimeSpan.FromSeconds(10);
+ private readonly TimeSpan _idleTimeout;
+ private readonly ITracer _tracer;
+ private DateTime _lastActivity;
+
+ public IdleManager(TimeSpan idleTimeout, ITracer tracer)
+ : this(idleTimeout, tracer, DateTime.UtcNow)
+ {
+ }
+
+ internal IdleManager(TimeSpan idleTimeout, ITracer tracer, DateTime dateTime)
+ {
+ _idleTimeout = idleTimeout;
+ _tracer = tracer;
+ _lastActivity = dateTime;
+ }
+
+ public void UpdateActivity()
+ {
+ _lastActivity = DateTime.UtcNow;
+ }
+
+ public void WaitForExit(Process process)
+ {
+ var processWrapper = new ProcessWrapper(process);
+ WaitForExit(processWrapper);
+ }
+
+ internal void WaitForExit(IProcess process)
+ {
+ // For the duration of the idle timeout, do nothing. Simply wait for the process to execute.
+ if (!process.WaitForExit(_idleTimeout))
+ {
+ long previousCpuUsage = process.GetTotalProcessorTime();
+
+ TimeSpan totalWaitDuration = _idleTimeout;
+ while (!process.WaitForExit(WaitIntervalTimeSpan))
+ {
+ totalWaitDuration += WaitIntervalTimeSpan;
+ if (totalWaitDuration >= Constants.MaxAllowedExecutionTime)
+ {
+ process.Kill(_tracer);
+
+ // The duration a process is executing is capped. If it exceeds this period, we'll kill it regardless of it actually performing any activity.
+ ThrowIdleTimeoutException(process, totalWaitDuration);
+ }
+
+ // Did we see any IO activity during the last wait interval?
+ if (DateTime.UtcNow > _lastActivity.Add(WaitIntervalTimeSpan))
+ {
+ // There wasn't any IO activity. Check if we had any CPU activity
+ long currentCpuUsage = process.GetTotalProcessorTime();
+ if (currentCpuUsage != previousCpuUsage)
+ {
+ // The process performed some compute bound operation. We'll wait for it some more
+ previousCpuUsage = currentCpuUsage;
+ continue;
+ }
+
+ // It's likely that the process is idling waiting for user input. Kill it
+ process.Kill(_tracer);
+ ThrowIdleTimeoutException(process, totalWaitDuration);
+ break;
+ }
+ }
+ }
+
+ process.WaitUntilEOF();
+ }
+
+ private static void ThrowIdleTimeoutException(IProcess process, TimeSpan totalWaitDuration)
+ {
+ string arguments = (process.Arguments ?? String.Empty).Trim();
+ if (arguments.Length > 15)
+ {
+ arguments = arguments.Substring(0, 15) + " ...";
+ }
+ string message = String.Format(Resources.Error_ProcessAborted, process.Name + " " + arguments, totalWaitDuration.TotalSeconds);
+ throw new CommandLineException(process.Name, process.Arguments, message)
+ {
+ ExitCode = -1,
+ Output = message,
+ Error = message
+ };
+ }
+ }
+}
View
9 Kudu.Core/Infrastructure/ProcessExtensions.cs
@@ -44,6 +44,15 @@ public static IEnumerable<Process> GetChildren(this Process process)
return Enumerable.Empty<Process>();
}
+ /// <summary>
+ /// Calculates the sum of TotalProcessorTime for the current process and all its children.
+ /// </summary>
+ public static long GetTotalProcessorTime(this Process process)
+ {
+ return new[] { process }.Concat(process.GetChildren())
+ .Sum(p => p.TotalProcessorTime.Ticks);
+ }
+
private static Process SafeGetProcessById(int pid)
{
try
View
47 Kudu.Core/Infrastructure/ProcessWrapper.cs
@@ -0,0 +1,47 @@
+using System;
+using System.Diagnostics;
+using System.IO;
+
+namespace Kudu.Core.Infrastructure
+{
+ public class ProcessWrapper : IProcess
+ {
+ private readonly Process _process;
+
+ public ProcessWrapper(Process process)
+ {
+ _process = process;
+ Name = Path.GetFileName(process.StartInfo.FileName);
+ }
+
+ public string Name { get; private set; }
+
+ public string Arguments
+ {
+ get
+ {
+ return _process.StartInfo.Arguments;
+ }
+ }
+
+ public void WaitUntilEOF()
+ {
+ _process.WaitForExit(-1);
+ }
+
+ public bool WaitForExit(TimeSpan timeSpan)
+ {
+ return _process.WaitForExit((int)timeSpan.TotalMilliseconds);
+ }
+
+ public void Kill(Contracts.Tracing.ITracer tracer)
+ {
+ _process.Kill(includesChildren: true, tracer: tracer);
+ }
+
+ public long GetTotalProcessorTime()
+ {
+ return _process.GetTotalProcessorTime();
+ }
+ }
+}
View
3  Kudu.Core/Kudu.Core.csproj
@@ -73,10 +73,13 @@
<Compile Include="Deployment\NpmExecutable.cs" />
<Compile Include="Deployment\NullLogger.cs" />
<Compile Include="Deployment\WellKnownEnvironmentVariables.cs" />
+ <Compile Include="Infrastructure\IdleManager.cs" />
<Compile Include="Infrastructure\CommandLineException.cs" />
<Compile Include="Infrastructure\ExecutableExtensions.cs" />
<Compile Include="Infrastructure\IExecutable.cs" />
+ <Compile Include="Infrastructure\IProcess.cs" />
<Compile Include="Infrastructure\ProcessExtensions.cs" />
+ <Compile Include="Infrastructure\ProcessWrapper.cs" />
<Compile Include="Infrastructure\ProgressWriter.cs" />
<Compile Include="Infrastructure\DisposableAction.cs" />
<Compile Include="Infrastructure\IniFile.cs" />
View
4 Kudu.Core/Resources.Designer.cs
@@ -1,7 +1,7 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
-// Runtime Version:4.0.30319.18033
+// Runtime Version:4.0.30319.18010
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
@@ -169,7 +169,7 @@ internal class Resources {
}
/// <summary>
- /// Looks up a localized string similar to Process &apos;{0}&apos; aborted due to idle timeout..
+ /// Looks up a localized string similar to Command &apos;{0}&apos; aborted due to idle timeout after &apos;{1}&apos; seconds..
/// </summary>
internal static string Error_ProcessAborted {
get {
View
2  Kudu.Core/Resources.resx
@@ -154,7 +154,7 @@
<value>Private Key already exists</value>
</data>
<data name="Error_ProcessAborted" xml:space="preserve">
- <value>Process '{0}' aborted due to idle timeout.</value>
+ <value>Command '{0}' aborted due to idle timeout after '{1}' seconds.</value>
</data>
<data name="Error_ProjectDoesNotExist" xml:space="preserve">
<value>The specified project '{0}' is not valid. It needs to point to either a csproj/vbproj file or to a directory.</value>
View
2  Kudu.Core/SourceControl/Git/GitExeServer.cs
@@ -16,7 +16,7 @@ public class GitExeServer : IGitServer
// Server git operations like receive-pack can take a long time for large repros, without any data flowing.
// So use a long 30 minute timeout here instead of the much shorter default.
- private static readonly TimeSpan _gitMinTimeout = TimeSpan.FromMinutes(30);
+ private static readonly TimeSpan _gitMinTimeout = Constants.MaxAllowedExecutionTime;
private readonly GitExecutable _gitExe;
private readonly ITraceFactory _traceFactory;
View
2  Kudu.FunctionalTests/GitRepositoryManagementTests.cs
@@ -968,7 +968,7 @@ public void HangProcessTest()
Assert.Contains("remote: Sleep(4000)", trace);
Assert.Contains("remote: Sleep(6000)", trace);
Assert.DoesNotContain("remote: Sleep(30000)", trace);
- Assert.Contains("remote: Process 'starter.cmd' aborted due to idle timeout.", trace);
+ Assert.Contains("remote: Command 'starter.cmd simplesleep.exe ...' aborted due to idle timeout after", trace);
// in certain OS, the child process killed may not work
// this only intends for public Kudu (test running on the same machine as git server).
View
19 Kudu.SiteManagement/IdleManager.cs
@@ -0,0 +1,19 @@
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Kudu.SiteManagement
+{
+ internal class IdleManager
+ {
+ [SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic", Justification = "This mirrors the signature of Kudu.Core.IdleManager")]
+ public void UpdateActivity()
+ {
+ }
+
+ [SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic", Justification = "This mirrors the signature of Kudu.Core.IdleManager")]
+ public void WaitForExit(Process process)
+ {
+ process.WaitForExit();
+ }
+ }
+}
View
1  Kudu.SiteManagement/Kudu.SiteManagement.csproj
@@ -55,6 +55,7 @@
<Link>OperationManager.cs</Link>
</Compile>
<Compile Include="DefaultSettingsResolver.cs" />
+ <Compile Include="IdleManager.cs" />
<Compile Include="ISettingsResolver.cs" />
<Compile Include="DefaultPathResolver.cs" />
<Compile Include="IISExtensions.cs" />
Please sign in to comment.
Something went wrong with that request. Please try again.