From e600de93b78eb45335947741232292ea60c1530e Mon Sep 17 00:00:00 2001 From: Amanda Deel <52223332+amdeel@users.noreply.github.com> Date: Wed, 25 Nov 2020 09:26:47 -0800 Subject: [PATCH] Added RestartAsync API to rerun existing orchestrator instances (#1545) --- .../ContextImplementations/DurableClient.cs | 13 ++ .../IDurableOrchestrationClient.cs | 10 + .../HttpApiHandler.cs | 111 ++++++++--- .../HttpManagementPayload.cs | 9 + ....WebJobs.Extensions.DurableTask-net461.xml | 18 ++ ...t.Azure.WebJobs.Extensions.DurableTask.xml | 18 ++ test/Common/DurableTaskEndToEndTests.cs | 77 +++++++- test/Common/HttpApiHandlerTests.cs | 179 +++++++++++++++++- 8 files changed, 408 insertions(+), 27 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableClient.cs b/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableClient.cs index a21e64b3f..3bd440ee9 100644 --- a/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableClient.cs +++ b/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableClient.cs @@ -911,6 +911,19 @@ Task IDurableOrchestrationClient.StartNewAsync(string orchestratorFun return ((IDurableOrchestrationClient)this).StartNewAsync(orchestratorFunctionName, string.Empty, input); } + async Task IDurableOrchestrationClient.RestartAsync(string instanceId, bool restartWithNewInstanceId) + { + DurableOrchestrationStatus status = await ((IDurableOrchestrationClient)this).GetStatusAsync(instanceId, showHistory: false, showHistoryOutput: false, showInput: true); + + if (status == null) + { + throw new ArgumentException($"An orchestrastion with the instanceId {instanceId} was not found."); + } + + return restartWithNewInstanceId ? await ((IDurableOrchestrationClient)this).StartNewAsync(orchestratorFunctionName: status.Name, status.Input) + : await ((IDurableOrchestrationClient)this).StartNewAsync(orchestratorFunctionName: status.Name, instanceId: status.InstanceId, status.Input); + } + /// Task IDurableEntityClient.SignalEntityAsync(string entityKey, Action operation) { diff --git a/src/WebJobs.Extensions.DurableTask/ContextInterfaces/IDurableOrchestrationClient.cs b/src/WebJobs.Extensions.DurableTask/ContextInterfaces/IDurableOrchestrationClient.cs index 5055bfe07..9ddea124d 100644 --- a/src/WebJobs.Extensions.DurableTask/ContextInterfaces/IDurableOrchestrationClient.cs +++ b/src/WebJobs.Extensions.DurableTask/ContextInterfaces/IDurableOrchestrationClient.cs @@ -288,5 +288,15 @@ public interface IDurableOrchestrationClient /// Cancellation token that can be used to cancel the status query operation. /// Returns each page of orchestration status for all instances and continuation token of next page. Task ListInstancesAsync(OrchestrationStatusQueryCondition condition, CancellationToken cancellationToken); + + /// + /// Restarts an existing orchestrator with the original input. + /// + /// InstanceId of a previously run orchestrator to restart. + /// Optional parameter that configures if restarting an orchestration will use a new instanceId or if it will + /// reuse the old instanceId. Defauls to true. + /// A task that completes when the orchestration is started. The task contains the instance id of the started + /// orchestratation instance. + Task RestartAsync(string instanceId, bool restartWithNewInstanceId = true); } } diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index 488c20170..e79f5cae1 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -41,6 +41,7 @@ internal class HttpApiHandler : IDisposable private const string RaiseEventOperation = "raiseEvent"; private const string TerminateOperation = "terminate"; private const string RewindOperation = "rewind"; + private const string RestartOperation = "restart"; private const string ShowHistoryParameter = "showHistory"; private const string ShowHistoryOutputParameter = "showHistoryOutput"; private const string ShowInputParameter = "showInput"; @@ -52,6 +53,9 @@ internal class HttpApiHandler : IDisposable private const string ReturnInternalServerErrorOnFailure = "returnInternalServerErrorOnFailure"; private const string LastOperationTimeFrom = "lastOperationTimeFrom"; private const string LastOperationTimeTo = "lastOperationTimeTo"; + private const string RestartWithNewInstanceId = "restartWithNewInstanceId"; + private const string TimeoutParameter = "timeout"; + private const string PollingInterval = "pollingInterval"; private const string EmptyEntityKeySymbol = "$"; @@ -110,7 +114,8 @@ public void Dispose() httpManagementPayload.StatusQueryGetUri, httpManagementPayload.SendEventPostUri, httpManagementPayload.TerminatePostUri, - httpManagementPayload.PurgeHistoryDeleteUri); + httpManagementPayload.PurgeHistoryDeleteUri, + httpManagementPayload.RestartUri); } // /orchestrators/{functionName}/{instanceId?} @@ -155,13 +160,16 @@ private static TemplateMatcher GetInstanceRaiseEventRoute() internal HttpManagementPayload CreateHttpManagementPayload( string instanceId, string taskHub, - string connectionName) + string connectionName, + bool returnInternalServerErrorOnFailure = false, + bool restartWithNewInstanceId = true) { HttpManagementPayload httpManagementPayload = this.GetClientResponseLinks( null, instanceId, taskHub, - connectionName); + connectionName, + returnInternalServerErrorOnFailure); return httpManagementPayload; } @@ -239,7 +247,8 @@ private static TemplateMatcher GetInstanceRaiseEventRoute() httpManagementPayload.StatusQueryGetUri, httpManagementPayload.SendEventPostUri, httpManagementPayload.TerminatePostUri, - httpManagementPayload.PurgeHistoryDeleteUri); + httpManagementPayload.PurgeHistoryDeleteUri, + httpManagementPayload.RestartUri); } } } @@ -353,7 +362,7 @@ public async Task HandleRequestAsync(HttpRequestMessage req return request.CreateResponse(HttpStatusCode.NotFound); } } - else + else if (request.Method == HttpMethod.Post) { if (string.Equals(operation, TerminateOperation, StringComparison.OrdinalIgnoreCase)) { @@ -363,11 +372,15 @@ public async Task HandleRequestAsync(HttpRequestMessage req { return await this.HandleRewindInstanceRequestAsync(request, instanceId); } - else + else if (string.Equals(operation, RestartOperation, StringComparison.OrdinalIgnoreCase)) { - return request.CreateResponse(HttpStatusCode.NotFound); + return await this.HandleRestartInstanceRequestAsync(request, instanceId); } } + else + { + return request.CreateResponse(HttpStatusCode.NotFound); + } } if (InstanceRaiseEventRoute.TryMatch(path, routeValues)) @@ -692,6 +705,18 @@ private static bool TryGetIntQueryParameterValue(NameValueCollection queryString return int.TryParse(value, out intValue); } + private static bool TryGetTimeSpanQueryParameterValue(NameValueCollection queryStringNameValueCollection, string queryParameterName, out TimeSpan? timeSpanValue) + { + timeSpanValue = null; + string value = queryStringNameValueCollection[queryParameterName]; + if (double.TryParse(value, out double doubleValue)) + { + timeSpanValue = TimeSpan.FromSeconds(doubleValue); + } + + return timeSpanValue != null; + } + private async Task HandleTerminateInstanceRequestAsync( HttpRequestMessage request, string instanceId) @@ -729,6 +754,8 @@ private static bool TryGetIntQueryParameterValue(NameValueCollection queryString { IDurableOrchestrationClient client = this.GetClient(request); + var queryNameValuePairs = request.GetQueryNameValuePairs(); + object input = null; if (request.Content != null && request.Content.Headers?.ContentLength != 0) { @@ -738,8 +765,8 @@ private static bool TryGetIntQueryParameterValue(NameValueCollection queryString string id = await client.StartNewAsync(functionName, instanceId, input); - TimeSpan? timeout = GetTimeSpan(request, "timeout"); - TimeSpan? pollingInterval = GetTimeSpan(request, "pollingInterval"); + TryGetTimeSpanQueryParameterValue(queryNameValuePairs, TimeoutParameter, out TimeSpan? timeout); + TryGetTimeSpanQueryParameterValue(queryNameValuePairs, PollingInterval, out TimeSpan? pollingInterval); // for durability providers that support poll-free waiting, we override the specified polling interval if (client is DurableClient durableClient && durableClient.DurabilityProvider.SupportsPollFreeWait) @@ -749,7 +776,7 @@ private static bool TryGetIntQueryParameterValue(NameValueCollection queryString if (timeout.HasValue && pollingInterval.HasValue) { - return await client.WaitForCompletionOrCreateCheckStatusResponseAsync(request, id, timeout.Value, pollingInterval.Value); + return await client.WaitForCompletionOrCreateCheckStatusResponseAsync(request, id, timeout, pollingInterval); } else { @@ -762,6 +789,54 @@ private static bool TryGetIntQueryParameterValue(NameValueCollection queryString } } + private async Task HandleRestartInstanceRequestAsync( + HttpRequestMessage request, + string instanceId) + { + try + { + IDurableOrchestrationClient client = this.GetClient(request); + + var queryNameValuePairs = request.GetQueryNameValuePairs(); + + string newInstanceId; + if (TryGetBooleanQueryParameterValue(queryNameValuePairs, RestartWithNewInstanceId, out bool restartWithNewInstanceId)) + { + newInstanceId = await client.RestartAsync(instanceId, restartWithNewInstanceId); + } + else + { + newInstanceId = await client.RestartAsync(instanceId); + } + + TryGetTimeSpanQueryParameterValue(queryNameValuePairs, TimeoutParameter, out TimeSpan? timeout); + TryGetTimeSpanQueryParameterValue(queryNameValuePairs, PollingInterval, out TimeSpan? pollingInterval); + + // for durability providers that support poll-free waiting, we override the specified polling interval + if (client is DurableClient durableClient && durableClient.DurabilityProvider.SupportsPollFreeWait) + { + pollingInterval = timeout; + } + + if (timeout.HasValue && pollingInterval.HasValue) + { + return await client.WaitForCompletionOrCreateCheckStatusResponseAsync(request, newInstanceId, timeout, pollingInterval); + } + else + { + return client.CreateCheckStatusResponse(request, newInstanceId); + } + } + catch (ArgumentException e) + { + return request.CreateErrorResponse(HttpStatusCode.BadRequest, "InstanceId does not match a valid orchestration instance.", e); + } + catch (JsonReaderException e) + { + return request.CreateErrorResponse(HttpStatusCode.BadRequest, "Invalid JSON content", e); + } + } + private async Task HandleRewindInstanceRequestAsync( HttpRequestMessage request, string instanceId) @@ -1017,6 +1092,7 @@ internal HttpCreationPayload GetInstanceCreationLinks() TerminatePostUri = instancePrefix + "/" + TerminateOperation + "?reason={text}&" + querySuffix, RewindPostUri = instancePrefix + "/" + RewindOperation + "?reason={text}&" + querySuffix, PurgeHistoryDeleteUri = instancePrefix + "?" + querySuffix, + RestartUri = instancePrefix + "/" + RestartOperation + "?" + querySuffix, }; if (returnInternalServerErrorOnFailure) @@ -1033,7 +1109,8 @@ internal HttpCreationPayload GetInstanceCreationLinks() string statusQueryGetUri, string sendEventPostUri, string terminatePostUri, - string purgeHistoryDeleteUri) + string purgeHistoryDeleteUri, + string restartUri) { HttpResponseMessage response = request.CreateResponse( HttpStatusCode.Accepted, @@ -1044,6 +1121,7 @@ internal HttpCreationPayload GetInstanceCreationLinks() sendEventPostUri, terminatePostUri, purgeHistoryDeleteUri, + restartUri, }); // Implement the async HTTP 202 pattern. @@ -1104,16 +1182,5 @@ internal async Task StopLocalHttpServerAsync() } } #endif - - private static TimeSpan? GetTimeSpan(HttpRequestMessage request, string queryParameterName) - { - string queryParameterStringValue = request.GetQueryNameValuePairs()[queryParameterName]; - if (string.IsNullOrEmpty(queryParameterStringValue)) - { - return null; - } - - return TimeSpan.FromSeconds(double.Parse(queryParameterStringValue)); - } } } diff --git a/src/WebJobs.Extensions.DurableTask/HttpManagementPayload.cs b/src/WebJobs.Extensions.DurableTask/HttpManagementPayload.cs index 604fc8fed..68cbbb92f 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpManagementPayload.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpManagementPayload.cs @@ -63,5 +63,14 @@ public class HttpManagementPayload /// [JsonProperty("purgeHistoryDeleteUri")] public string PurgeHistoryDeleteUri { get; internal set; } + + /// + /// Gets the HTTP POST instance restart endpoint. + /// + /// + /// The HTTP URL for restarting an orchestration instance. + /// + [JsonProperty("restartUri")] + public string RestartUri { get; internal set; } } } diff --git a/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask-net461.xml b/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask-net461.xml index 003fd5bc3..6035ca8e9 100644 --- a/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask-net461.xml +++ b/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask-net461.xml @@ -962,6 +962,16 @@ Cancellation token that can be used to cancel the status query operation. Returns each page of orchestration status for all instances and continuation token of next page. + + + Restarts an existing orchestrator with the original input. + + InstanceId of a previously run orchestrator to restart. + Optional parameter that configures if restarting an orchestration will use a new instanceId or if it will + reuse the old instanceId. Defauls to true. + A task that completes when the orchestration is started. The task contains the instance id of the started + orchestratation instance. + Provides functionality available to orchestration code. @@ -2846,6 +2856,14 @@ The HTTP URL for purging instance history by instance ID. + + + Gets the HTTP POST instance restart endpoint. + + + The HTTP URL for restarting an orchestration instance. + + Custom service interface for signaling the extension when the function app is starting up or shutting down. diff --git a/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml b/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml index 9e8e87a91..7de0bf855 100644 --- a/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml +++ b/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml @@ -967,6 +967,16 @@ Cancellation token that can be used to cancel the status query operation. Returns each page of orchestration status for all instances and continuation token of next page. + + + Restarts an existing orchestrator with the original input. + + InstanceId of a previously run orchestrator to restart. + Optional parameter that configures if restarting an orchestration will use a new instanceId or if it will + reuse the old instanceId. Defauls to true. + A task that completes when the orchestration is started. The task contains the instance id of the started + orchestratation instance. + Provides functionality available to orchestration code. @@ -2872,6 +2882,14 @@ The HTTP URL for purging instance history by instance ID. + + + Gets the HTTP POST instance restart endpoint. + + + The HTTP URL for restarting an orchestration instance. + + Custom service interface for signaling the extension when the function app is starting up or shutting down. diff --git a/test/Common/DurableTaskEndToEndTests.cs b/test/Common/DurableTaskEndToEndTests.cs index 9454a3641..bb83d5023 100644 --- a/test/Common/DurableTaskEndToEndTests.cs +++ b/test/Common/DurableTaskEndToEndTests.cs @@ -4277,7 +4277,79 @@ public async Task Purge_Partially_History_By_TimePeriod(bool extendedSessions, s } } - [Theory(Skip = "Azure Storage fails due to container deletion")] + [Theory] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + [InlineData(true)] + [InlineData(false)] + public async Task RestartOrchestator_IsSuccess(bool restartWithNewInstanceId) + { + string[] orchestratorFunctionNames = + { + nameof(TestOrchestrations.SayHelloInline), + }; + using (var host = TestHelpers.GetJobHost( + this.loggerProvider, + nameof(this.HelloWorldOrchestration_Inline), + false)) + { + await host.StartAsync(); + + var instanceId = Guid.NewGuid().ToString(); + + var client = await host.StartOrchestratorAsync(orchestratorFunctionNames[0], "RestartAsyncTest", this.output, instanceId: instanceId); + await client.WaitForCompletionAsync(this.output); + + var newInstanceId = await client.InnerClient.RestartAsync(instanceId, restartWithNewInstanceId: restartWithNewInstanceId); + var status = await client.WaitForCompletionAsync(this.output); + + if (restartWithNewInstanceId) + { + Assert.NotEqual(instanceId, newInstanceId); + } + else + { + Assert.Equal(instanceId, newInstanceId); + } + + Assert.Equal(OrchestrationRuntimeStatus.Completed, status?.RuntimeStatus); + Assert.Equal("RestartAsyncTest", status?.Input); + + await host.StopAsync(); + } + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task RestartOrchestrator_ThrowsException() + { + string[] orchestratorFunctionNames = + { + nameof(TestOrchestrations.SayHelloInline), + }; + using (var host = TestHelpers.GetJobHost( + this.loggerProvider, + nameof(this.HelloWorldOrchestration_Inline), + false)) + { + await host.StartAsync(); + + var nonExistentId = Guid.NewGuid().ToString(); + + var client = await host.StartOrchestratorAsync(orchestratorFunctionNames[0], "World", this.output); + + ArgumentException exception = + await Assert.ThrowsAsync(async () => + { + await client.InnerClient.RestartAsync(nonExistentId); + }); + + Assert.Equal( + $"An orchestrastion with the instanceId {nonExistentId} was not found.", + exception.Message); + } + } + + [Theory] [Trait("Category", PlatformSpecificHelpers.TestCategory)] [MemberData(nameof(TestDataGenerator.GetBooleanAndFullFeaturedStorageProviderOptions), MemberType = typeof(TestDataGenerator))] public async Task GetStatus_WithCondition(bool extendedSessions, string storageProvider) @@ -5331,6 +5403,9 @@ private static void ValidateHttpManagementPayload(HttpManagementPayload httpMana Assert.Equal( $"{notificationUrl}/instances/{instanceId}/terminate?reason={{text}}&taskHub={taskHubName}&connection=AzureWebJobsStorage&code=mykey", httpManagementPayload.TerminatePostUri); + Assert.Equal( + $"{notificationUrl}/instances/{instanceId}/restart?taskHub={taskHubName}&connection=AzureWebJobsStorage&code=mykey", + httpManagementPayload.RestartUri); } [DataContract] diff --git a/test/Common/HttpApiHandlerTests.cs b/test/Common/HttpApiHandlerTests.cs index ec16ccfc6..9dac8ffec 100644 --- a/test/Common/HttpApiHandlerTests.cs +++ b/test/Common/HttpApiHandlerTests.cs @@ -95,6 +95,9 @@ public async Task CreateCheckStatusResponse_Returns_Correct_HTTP_202_Response() Assert.Equal( $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742?taskHub=SampleHubVS&connection=Storage&code=mykey", (string)status["purgeHistoryDeleteUri"]); + Assert.Equal( + $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742/restart?taskHub=SampleHubVS&connection=Storage&code=mykey", + (string)status["restartUri"]); } [Fact] @@ -117,6 +120,9 @@ public void CreateCheckStatus_Returns_Correct_HttpManagementPayload_based_on_def Assert.Equal( $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742?taskHub=DurableFunctionsHub&connection=Storage&code=mykey", httpManagementPayload.PurgeHistoryDeleteUri); + Assert.Equal( + $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742/restart?taskHub=DurableFunctionsHub&connection=Storage&code=mykey", + httpManagementPayload.RestartUri); } [Fact] @@ -139,6 +145,9 @@ public void CreateCheckStatus_Returns_Correct_HttpManagementPayload_based_on_cus Assert.Equal( $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742?taskHub=SampleHubVS&connection=Storage&code=mykey", httpManagementPayload.PurgeHistoryDeleteUri); + Assert.Equal( + $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742/restart?taskHub=SampleHubVS&connection=Storage&code=mykey", + httpManagementPayload.RestartUri); } [Fact] @@ -161,6 +170,9 @@ public void CreateCheckStatus_Returns_Correct_HttpManagementPayload_based_on_cus Assert.Equal( $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742?taskHub=DurableFunctionsHub&connection=TestConnection&code=mykey", httpManagementPayload.PurgeHistoryDeleteUri); + Assert.Equal( + $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742/restart?taskHub=DurableFunctionsHub&connection=TestConnection&code=mykey", + httpManagementPayload.RestartUri); } [Fact] @@ -168,11 +180,11 @@ public void CreateCheckStatus_Returns_Correct_HttpManagementPayload_based_on_cus public void CreateCheckStatus_Returns_Correct_HttpManagementPayload_based_on_custom_values() { var httpApiHandler = new HttpApiHandler(GetTestExtension(), null); - HttpManagementPayload httpManagementPayload = httpApiHandler.CreateHttpManagementPayload(TestConstants.InstanceId, TestConstants.TaskHub, TestConstants.CustomConnectionName); + HttpManagementPayload httpManagementPayload = httpApiHandler.CreateHttpManagementPayload(TestConstants.InstanceId, TestConstants.TaskHub, TestConstants.CustomConnectionName, returnInternalServerErrorOnFailure: true, restartWithNewInstanceId: false); Assert.NotNull(httpManagementPayload); Assert.Equal(httpManagementPayload.Id, TestConstants.InstanceId); Assert.Equal( - $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742?taskHub=SampleHubVS&connection=TestConnection&code=mykey", + $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742?taskHub=SampleHubVS&connection=TestConnection&code=mykey&returnInternalServerErrorOnFailure=true", httpManagementPayload.StatusQueryGetUri); Assert.Equal( $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742/raiseEvent/{{eventName}}?taskHub=SampleHubVS&connection=TestConnection&code=mykey", @@ -183,6 +195,9 @@ public void CreateCheckStatus_Returns_Correct_HttpManagementPayload_based_on_cus Assert.Equal( $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742?taskHub=SampleHubVS&connection=TestConnection&code=mykey", httpManagementPayload.PurgeHistoryDeleteUri); + Assert.Equal( + $"{TestConstants.NotificationUrlBase}/instances/7b59154ae666471993659902ed0ba742/restart?taskHub=SampleHubVS&connection=TestConnection&code=mykey", + httpManagementPayload.RestartUri); } [Fact] @@ -261,6 +276,9 @@ public async Task WaitForCompletionOrCreateCheckStatusResponseAsync_Returns_HTTP Assert.Equal( $"{TestConstants.NotificationUrlBase}/instances/9b59154ae666471993659902ed0ba749?taskHub=SampleHubVS&connection=Storage&code=mykey", (string)status["purgeHistoryDeleteUri"]); + Assert.Equal( + $"{TestConstants.NotificationUrlBase}/instances/9b59154ae666471993659902ed0ba749/restart?taskHub=SampleHubVS&connection=Storage&code=mykey", + (string)status["restartUri"]); Assert.True(stopWatch.Elapsed > TimeSpan.FromSeconds(30)); } @@ -743,6 +761,153 @@ public async Task TerminateInstanceWebhook() Assert.Equal(testReason, actualReason); } + [Theory] + [InlineData(true)] + [InlineData(false)] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task RestartInstance_Is_Success(bool restartWithNewInstanceId) + { + string testInstanceId = Guid.NewGuid().ToString(); + string restartedInstanceId = restartWithNewInstanceId ? Guid.NewGuid().ToString() : testInstanceId; + + var restartUriBuilder = new UriBuilder(TestConstants.NotificationUrl); + restartUriBuilder.Path += $"/Instances/{testInstanceId}/restart"; + restartUriBuilder.Query = $"restartWithNewInstanceId={restartWithNewInstanceId}&{restartUriBuilder.Query.TrimStart('?')}"; + + var testRequest = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = restartUriBuilder.Uri, + }; + + var testStatusQueryGetUri = $"{TestConstants.NotificationUrlBase}/instances/{restartedInstanceId}?taskhub=SampleHubVS&connection=Storage&code=mykey"; + var testSendEventPostUri = $"{TestConstants.NotificationUrlBase}/instances/{restartedInstanceId}/raiseEvent/{{eventName}}?taskHub=SampleHubVS&connection=Storage&code=mykey"; + var testTerminatePostUri = $"{TestConstants.NotificationUrlBase}/instances/{restartedInstanceId}/terminate?reason={{text}}&taskHub=SampleHubVS&connection=Storage&code=mykey"; + var testRewindPostUri = $"{TestConstants.NotificationUrlBase}/instances/{restartedInstanceId}/rewind?reason={{text}}&taskHub=SampleHubVS&connection=Storage&code=mykey"; + var testRestartPostUri = $"{TestConstants.NotificationUrlBase}/instances/{restartedInstanceId}/restart?taskHub=SampleHubVS&connection=Storage&code=mykey&restartWithNewInstanceId={restartWithNewInstanceId}"; + var testResponse = testRequest.CreateResponse( + HttpStatusCode.Accepted, + new + { + id = restartedInstanceId, + statusQueryGetUri = testStatusQueryGetUri, + sendEventPostUri = testSendEventPostUri, + terminatePostUri = testTerminatePostUri, + rewindPostUri = testRewindPostUri, + restartPostUri = testRestartPostUri, + }); + + var clientMock = new Mock(); + clientMock + .Setup(x => x.RestartAsync(testInstanceId, restartWithNewInstanceId)) + .Returns(Task.FromResult(restartedInstanceId)); + + clientMock + .Setup(x => x.CreateCheckStatusResponse(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(testResponse); + + var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest); + + Assert.Equal(HttpStatusCode.Accepted, actualResponse.StatusCode); + var content = await actualResponse.Content.ReadAsStringAsync(); + var status = JsonConvert.DeserializeObject(content); + Assert.Equal(status["id"], restartedInstanceId); + Assert.Equal(status["statusQueryGetUri"], testStatusQueryGetUri); + Assert.Equal(status["sendEventPostUri"], testSendEventPostUri); + Assert.Equal(status["terminatePostUri"], testTerminatePostUri); + Assert.Equal(status["rewindPostUri"], testRewindPostUri); + Assert.Equal(status["restartPostUri"], testRestartPostUri); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task RestartInstanceAndWaitToComplete_Is_Success(bool restartWithNewInstanceId) + { + string testInstanceId = Guid.NewGuid().ToString(); + string restartedInstanceId = restartWithNewInstanceId ? Guid.NewGuid().ToString() : testInstanceId; + + var restartUriBuilder = new UriBuilder(TestConstants.NotificationUrl); + restartUriBuilder.Path += $"/Instances/{testInstanceId}/restart"; + restartUriBuilder.Query = $"timeout=90&pollingInterval=10&restartWithNewInstanceId={restartWithNewInstanceId}&{restartUriBuilder.Query.TrimStart('?')}"; + + var testRequest = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = restartUriBuilder.Uri, + }; + + var testStatusQueryGetUri = $"{TestConstants.NotificationUrlBase}/instances/{restartedInstanceId}?taskhub=SampleHubVS&connection=Storage&code=mykey"; + var testSendEventPostUri = $"{TestConstants.NotificationUrlBase}/instances/{restartedInstanceId}/raiseEvent/{{eventName}}?taskHub=SampleHubVS&connection=Storage&code=mykey"; + var testTerminatePostUri = $"{TestConstants.NotificationUrlBase}/instances/{restartedInstanceId}/terminate?reason={{text}}&taskHub=SampleHubVS&connection=Storage&code=mykey"; + var testRewindPostUri = $"{TestConstants.NotificationUrlBase}/instances/{restartedInstanceId}/rewind?reason={{text}}&taskHub=SampleHubVS&connection=Storage&code=mykey"; + var testRestartPostUri = $"{TestConstants.NotificationUrlBase}/instances/{restartedInstanceId}/restart?taskHub=SampleHubVS&connection=Storage&code=mykey&restartWithNewInstanceId={restartWithNewInstanceId}"; + var testResponse = testRequest.CreateResponse( + HttpStatusCode.Accepted, + new + { + id = restartedInstanceId, + statusQueryGetUri = testStatusQueryGetUri, + sendEventPostUri = testSendEventPostUri, + terminatePostUri = testTerminatePostUri, + rewindPostUri = testRewindPostUri, + restartPostUri = testRestartPostUri, + }); + + var clientMock = new Mock(); + clientMock + .Setup(x => x.RestartAsync(testInstanceId, restartWithNewInstanceId)) + .Returns(Task.FromResult(restartedInstanceId)); + + clientMock + .Setup(x => x.WaitForCompletionOrCreateCheckStatusResponseAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(testResponse)); + + var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest); + + Assert.Equal(HttpStatusCode.Accepted, actualResponse.StatusCode); + var content = await actualResponse.Content.ReadAsStringAsync(); + var status = JsonConvert.DeserializeObject(content); + Assert.Equal(status["id"], restartedInstanceId); + Assert.Equal(status["statusQueryGetUri"], testStatusQueryGetUri); + Assert.Equal(status["sendEventPostUri"], testSendEventPostUri); + Assert.Equal(status["terminatePostUri"], testTerminatePostUri); + Assert.Equal(status["rewindPostUri"], testRewindPostUri); + Assert.Equal(status["restartPostUri"], testRestartPostUri); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task RestartInstance_Returns_HTTP_400_On_Invalid_InstanceId() + { + string testBadInstanceId = Guid.NewGuid().ToString("N"); + + var startRequestUriBuilder = new UriBuilder(TestConstants.NotificationUrl); + startRequestUriBuilder.Path += $"/Instances/{testBadInstanceId}/restart"; + + var testRequest = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = startRequestUriBuilder.Uri, + }; + + var clientMock = new Mock(); + clientMock + .Setup(x => x.RestartAsync(It.IsAny(), It.IsAny())) + .Throws(new ArgumentException()); + + var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest); + + Assert.Equal(HttpStatusCode.BadRequest, actualResponse.StatusCode); + var content = await actualResponse.Content.ReadAsStringAsync(); + var error = JsonConvert.DeserializeObject(content); + Assert.Equal("InstanceId does not match a valid orchestration instance.", error["Message"].ToString()); + } + [Theory] [InlineData(null, false)] [InlineData(null, true)] @@ -766,10 +931,11 @@ public async Task StartNewInstance_Is_Success(string instanceId, bool hasContent : new StringContent("\"TestContent\""), }; - var testStatusQueryGetUri = $"{TestConstants.NotificationUrlBase}/instances/{testInstanceId}?taskhub=SampleHubVS&connection=Storage&code=mykey&returnInternalServerErrorOnFailure=False"; + var testStatusQueryGetUri = $"{TestConstants.NotificationUrlBase}/instances/{testInstanceId}?taskhub=SampleHubVS&connection=Storage&code=mykey"; var testSendEventPostUri = $"{TestConstants.NotificationUrlBase}/instances/{testInstanceId}/raiseEvent/{{eventName}}?taskHub=SampleHubVS&connection=Storage&code=mykey"; var testTerminatePostUri = $"{TestConstants.NotificationUrlBase}/instances/{testInstanceId}/terminate?reason={{text}}&taskHub=SampleHubVS&connection=Storage&code=mykey"; var testRewindPostUri = $"{TestConstants.NotificationUrlBase}/instances/{testInstanceId}/rewind?reason={{text}}&taskHub=SampleHubVS&connection=Storage&code=mykey"; + var testRestartPostUri = $"{TestConstants.NotificationUrlBase}/instances/{testInstanceId}/restart?taskHub=SampleHubVS&connection=Storage&code=mykey"; var testResponse = testRequest.CreateResponse( HttpStatusCode.Accepted, new @@ -779,6 +945,7 @@ public async Task StartNewInstance_Is_Success(string instanceId, bool hasContent sendEventPostUri = testSendEventPostUri, terminatePostUri = testTerminatePostUri, rewindPostUri = testRewindPostUri, + restartPostUri = testRestartPostUri, }); var clientMock = new Mock(); @@ -801,6 +968,7 @@ public async Task StartNewInstance_Is_Success(string instanceId, bool hasContent Assert.Equal(status["sendEventPostUri"], testSendEventPostUri); Assert.Equal(status["terminatePostUri"], testTerminatePostUri); Assert.Equal(status["rewindPostUri"], testRewindPostUri); + Assert.Equal(status["restartPostUri"], testRestartPostUri); } [Theory] @@ -827,10 +995,11 @@ public async Task StartNewInstanceAndWaitToComplete_Is_Success(string instanceId : new StringContent("\"TestContent\""), }; - var testStatusQueryGetUri = $"{TestConstants.NotificationUrlBase}/instances/{testInstanceId}?taskhub=SampleHubVS&connection=Storage&code=mykey&returnInternalServerErrorOnFailure=False"; + var testStatusQueryGetUri = $"{TestConstants.NotificationUrlBase}/instances/{testInstanceId}?taskhub=SampleHubVS&connection=Storage&code=mykey"; var testSendEventPostUri = $"{TestConstants.NotificationUrlBase}/instances/{testInstanceId}/raiseEvent/{{eventName}}?taskHub=SampleHubVS&connection=Storage&code=mykey"; var testTerminatePostUri = $"{TestConstants.NotificationUrlBase}/instances/{testInstanceId}/terminate?reason={{text}}&taskHub=SampleHubVS&connection=Storage&code=mykey"; var testRewindPostUri = $"{TestConstants.NotificationUrlBase}/instances/{testInstanceId}/rewind?reason={{text}}&taskHub=SampleHubVS&connection=Storage&code=mykey"; + var testRestartPostUri = $"{TestConstants.NotificationUrlBase}/instances/{testInstanceId}/restart?taskHub=SampleHubVS&connection=Storage&code=mykey"; var testResponse = testRequest.CreateResponse( HttpStatusCode.Accepted, new @@ -840,6 +1009,7 @@ public async Task StartNewInstanceAndWaitToComplete_Is_Success(string instanceId sendEventPostUri = testSendEventPostUri, terminatePostUri = testTerminatePostUri, rewindPostUri = testRewindPostUri, + restartPostUri = testRestartPostUri, }); var clientMock = new Mock(); @@ -862,6 +1032,7 @@ public async Task StartNewInstanceAndWaitToComplete_Is_Success(string instanceId Assert.Equal(status["sendEventPostUri"], testSendEventPostUri); Assert.Equal(status["terminatePostUri"], testTerminatePostUri); Assert.Equal(status["rewindPostUri"], testRewindPostUri); + Assert.Equal(status["restartPostUri"], testRestartPostUri); } [Fact]