From ea8a520cbaad41a541f67ae23e017c50390d3a7f Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Mon, 23 Jan 2023 08:13:04 -1000 Subject: [PATCH 01/62] Migrate to the New Azure Storage SDKs (#763) --- .../Obsolete/LegacyTableEntityConverter.cs | 316 +++++++ .../Storage/StorageUriExtensionsTests.cs | 37 - .../StorageAccountDetailsTests.cs | 89 -- .../TableEntityConverterTests.cs | 484 +++++++++++ .../TimeoutHandlerTests.cs | 138 --- samples/Correlation.Samples/TestHelpers.cs | 9 +- samples/DurableTask.Samples/Program.cs | 2 +- .../AnalyticsEventSource.cs | 6 +- .../AzureStorageOrchestrationService.cs | 112 +-- ...zureStorageOrchestrationServiceSettings.cs | 91 +- .../DefaultStorageServiceClientProvider.cs | 40 + .../DurableTask.AzureStorage.csproj | 20 +- .../Http/LeaseTimeoutHttpPipelinePolicy.cs | 55 ++ .../Http/MonitoringHttpPipelinePolicy.cs | 43 + .../Http/ThrottlingHttpPipelinePolicy.cs | 75 ++ .../IStorageServiceClientProvider.cs | 41 + .../Logging/LogEvents.cs | 12 +- .../Logging/LogHelper.cs | 6 +- src/DurableTask.AzureStorage/MessageData.cs | 3 +- .../MessageManager.cs | 80 +- .../Messaging/ControlQueue.cs | 17 +- .../Messaging/MessageCollection.cs | 4 +- .../Messaging/OrchestrationSession.cs | 5 +- .../Messaging/Session.cs | 6 +- .../Messaging/TaskHubQueue.cs | 18 +- .../Messaging/WorkItemQueue.cs | 1 + .../DisconnectedPerformanceMonitor.cs | 80 +- .../OrchestrationInstanceStatus.cs | 9 +- .../OrchestrationSessionManager.cs | 17 +- .../Partitioning/AppLeaseManager.cs | 1 - .../{BlobLease.cs => BlobPartitionLease.cs} | 15 +- ...anager.cs => BlobPartitionLeaseManager.cs} | 117 ++- .../Partitioning/ILeaseManager.cs | 23 +- .../Partitioning/IPartitionManager.cs | 3 +- .../Partitioning/Lease.cs | 8 +- .../Partitioning/LeaseCollectionBalancer.cs | 8 +- .../Partitioning/LegacyPartitionManager.cs | 17 +- .../Partitioning/SafePartitionManager.cs | 23 +- .../SimpleBufferManager.cs | 3 +- .../Storage/AsyncPageableAsyncProjection.cs | 62 ++ .../Storage/AsyncPageableProjection.cs | 59 ++ .../Storage/AzureStorageClient.cs | 148 ++-- src/DurableTask.AzureStorage/Storage/Blob.cs | 153 ++-- .../Storage/BlobContainer.cs | 125 +-- .../Storage/ClientResponseExtensions.cs | 97 +++ .../Storage/DurableTaskStorageException.cs | 13 +- ...ultResponseInfo.cs => OperationContext.cs} | 16 +- src/DurableTask.AzureStorage/Storage/Queue.cs | 177 ++-- .../Storage/QueueMessage.cs | 49 -- .../Storage/StorageUriExtensions.cs | 72 -- src/DurableTask.AzureStorage/Storage/Table.cs | 268 ++---- .../Storage/TableQueryResponse.cs | 57 ++ ...esResponseInfo.cs => TableQueryResults.cs} | 18 +- .../Storage/TableTransactionResults.cs | 37 + .../Storage/TableTransactionResultsBuilder.cs | 45 + .../StorageAccountClientProvider.cs | 118 +++ .../StorageAccountDetails.cs | 98 --- .../StorageServiceClientProvider.cs | 262 ++++++ .../TimeoutHandler.cs | 138 --- .../Tracking/AzureTableTrackingStore.cs | 783 +++++++----------- .../Tracking/ITrackingStore.cs | 77 +- .../Tracking/InstanceStatus.cs | 7 +- .../InstanceStoreBackedTrackingStore.cs | 49 +- .../Tracking/ODataCondition.cs | 46 + .../Tracking/OrchestrationHistory.cs | 5 +- ...chestrationInstanceStatusQueryCondition.cs | 70 +- .../Tracking/TableEntityConverter.cs | 434 +++++----- .../Tracking/TrackingStoreBase.cs | 47 +- .../FunctionalTests.cs | 1 - ...StorageOrchestrationServiceSettingsTest.cs | 7 +- .../AzureStorageScaleTests.cs | 116 +-- .../AzureStorageScenarioTests.cs | 158 +--- .../AzureTableTrackingStoreTest.cs | 250 ++---- .../DurableTask.AzureStorage.Tests.csproj | 4 +- ...trationInstanceStatusQueryConditionTest.cs | 41 +- .../TestHelpers.cs | 7 +- test/DurableTask.Stress.Tests/Program.cs | 4 +- 77 files changed, 3332 insertions(+), 2820 deletions(-) create mode 100644 Test/DurableTask.AzureStorage.Tests/Obsolete/LegacyTableEntityConverter.cs delete mode 100644 Test/DurableTask.AzureStorage.Tests/Storage/StorageUriExtensionsTests.cs delete mode 100644 Test/DurableTask.AzureStorage.Tests/StorageAccountDetailsTests.cs create mode 100644 Test/DurableTask.AzureStorage.Tests/TableEntityConverterTests.cs delete mode 100644 Test/DurableTask.AzureStorage.Tests/TimeoutHandlerTests.cs create mode 100644 src/DurableTask.AzureStorage/DefaultStorageServiceClientProvider.cs create mode 100644 src/DurableTask.AzureStorage/Http/LeaseTimeoutHttpPipelinePolicy.cs create mode 100644 src/DurableTask.AzureStorage/Http/MonitoringHttpPipelinePolicy.cs create mode 100644 src/DurableTask.AzureStorage/Http/ThrottlingHttpPipelinePolicy.cs create mode 100644 src/DurableTask.AzureStorage/IStorageServiceClientProvider.cs rename src/DurableTask.AzureStorage/Partitioning/{BlobLease.cs => BlobPartitionLease.cs} (74%) rename src/DurableTask.AzureStorage/Partitioning/{BlobLeaseManager.cs => BlobPartitionLeaseManager.cs} (72%) create mode 100644 src/DurableTask.AzureStorage/Storage/AsyncPageableAsyncProjection.cs create mode 100644 src/DurableTask.AzureStorage/Storage/AsyncPageableProjection.cs create mode 100644 src/DurableTask.AzureStorage/Storage/ClientResponseExtensions.cs rename src/DurableTask.AzureStorage/Storage/{TableResultResponseInfo.cs => OperationContext.cs} (72%) delete mode 100644 src/DurableTask.AzureStorage/Storage/QueueMessage.cs delete mode 100644 src/DurableTask.AzureStorage/Storage/StorageUriExtensions.cs create mode 100644 src/DurableTask.AzureStorage/Storage/TableQueryResponse.cs rename src/DurableTask.AzureStorage/Storage/{TableEntitiesResponseInfo.cs => TableQueryResults.cs} (61%) create mode 100644 src/DurableTask.AzureStorage/Storage/TableTransactionResults.cs create mode 100644 src/DurableTask.AzureStorage/Storage/TableTransactionResultsBuilder.cs create mode 100644 src/DurableTask.AzureStorage/StorageAccountClientProvider.cs delete mode 100644 src/DurableTask.AzureStorage/StorageAccountDetails.cs create mode 100644 src/DurableTask.AzureStorage/StorageServiceClientProvider.cs delete mode 100644 src/DurableTask.AzureStorage/TimeoutHandler.cs create mode 100644 src/DurableTask.AzureStorage/Tracking/ODataCondition.cs diff --git a/Test/DurableTask.AzureStorage.Tests/Obsolete/LegacyTableEntityConverter.cs b/Test/DurableTask.AzureStorage.Tests/Obsolete/LegacyTableEntityConverter.cs new file mode 100644 index 000000000..5347fffa7 --- /dev/null +++ b/Test/DurableTask.AzureStorage.Tests/Obsolete/LegacyTableEntityConverter.cs @@ -0,0 +1,316 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.AzureStorage.Tests.Obsolete +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Diagnostics; + using System.Reflection; + using System.Runtime.Serialization; + using Microsoft.WindowsAzure.Storage.Table; + + /// + /// Utility class for converting [DataContract] objects into DynamicTableEntity and back. + /// This class makes heavy use of reflection to build the entity converters. + /// + /// + /// This class is safe for concurrent usage by multiple threads. + /// + class LegacyTableEntityConverter + { + readonly ConcurrentDictionary> converterCache; + + public LegacyTableEntityConverter() + { + this.converterCache = new ConcurrentDictionary>(); + } + + /// + /// Converts a data contract object into a . + /// + public DynamicTableEntity ConvertToTableEntity(object obj) + { + if (obj == null) + { + throw new ArgumentNullException(nameof(obj)); + } + + Debug.Assert(obj.GetType().GetCustomAttribute() != null); + + IReadOnlyList propertyConverters = this.converterCache.GetOrAdd( + obj.GetType(), + GetPropertyConvertersForType); + + var tableEntity = new DynamicTableEntity(); + foreach (PropertyConverter propertyConverter in propertyConverters) + { + tableEntity.Properties[propertyConverter.PropertyName] = propertyConverter.GetEntityProperty(obj); + } + + return tableEntity; + } + + public object ConvertFromTableEntity(DynamicTableEntity tableEntity, Func typeFactory) + { + if (tableEntity == null) + { + throw new ArgumentNullException(nameof(tableEntity)); + } + + if (typeFactory == null) + { + throw new ArgumentNullException(nameof(typeFactory)); + } + + Type objectType = typeFactory(tableEntity); + object createdObject = FormatterServices.GetUninitializedObject(objectType); + + IReadOnlyList propertyConverters = this.converterCache.GetOrAdd( + objectType, + GetPropertyConvertersForType); + + foreach (PropertyConverter propertyConverter in propertyConverters) + { + // Properties with null values are not actually saved/retrieved by table storage. + EntityProperty entityProperty; + if (tableEntity.Properties.TryGetValue(propertyConverter.PropertyName, out entityProperty)) + { + propertyConverter.SetObjectProperty(createdObject, entityProperty); + } + } + + return createdObject; + } + + static List GetPropertyConvertersForType(Type type) + { + var propertyConverters = new List(); + BindingFlags flags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly; + + // Loop through the type hierarchy to find all [DataMember] attributes which belong to [DataContract] classes. + while (type != null && type.GetCustomAttribute() != null) + { + foreach (MemberInfo member in type.GetMembers(flags)) + { + DataMemberAttribute dataMember = member.GetCustomAttribute(); + if (dataMember == null) + { + continue; + } + + PropertyInfo property = member as PropertyInfo; + FieldInfo field = member as FieldInfo; + if (property == null && field == null) + { + throw new InvalidDataContractException("Only fields and properties can be marked as [DataMember]."); + } + else if (property != null && (!property.CanWrite || !property.CanRead)) + { + throw new InvalidDataContractException("[DataMember] properties must be both readable and writeable."); + } + + // Timestamp is a reserved property name in Table Storage, so the name needs to be changed. + string propertyName = dataMember.Name ?? member.Name; + if (string.Equals(propertyName, "Timestamp", StringComparison.OrdinalIgnoreCase)) + { + propertyName = "_Timestamp"; + } + + Func getEntityPropertyFunc; + Action setObjectPropertyFunc; + + Type memberValueType = property != null ? property.PropertyType : field.FieldType; + if (typeof(string).IsAssignableFrom(memberValueType)) + { + if (property != null) + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForString((string)property.GetValue(o)); + setObjectPropertyFunc = (o, e) => property.SetValue(o, e.StringValue); + } + else + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForString((string)field.GetValue(o)); + setObjectPropertyFunc = (o, e) => field.SetValue(o, e.StringValue); + } + } + else if (memberValueType.IsEnum) + { + // Enums are serialized as strings for readability. + if (property != null) + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForString(property.GetValue(o).ToString()); + setObjectPropertyFunc = (o, e) => property.SetValue(o, Enum.Parse(memberValueType, e.StringValue)); + } + else + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForString(field.GetValue(o).ToString()); + setObjectPropertyFunc = (o, e) => field.SetValue(o, Enum.Parse(memberValueType, e.StringValue)); + } + } + else if (typeof(int?).IsAssignableFrom(memberValueType)) + { + if (property != null) + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForInt((int?)property.GetValue(o)); + setObjectPropertyFunc = (o, e) => property.SetValue(o, e.Int32Value); + } + else + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForInt((int?)field.GetValue(o)); + setObjectPropertyFunc = (o, e) => field.SetValue(o, e.Int32Value); + } + } + else if (typeof(long?).IsAssignableFrom(memberValueType)) + { + if (property != null) + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForLong((long?)property.GetValue(o)); + setObjectPropertyFunc = (o, e) => property.SetValue(o, e.Int64Value); + } + else + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForLong((long?)field.GetValue(o)); + setObjectPropertyFunc = (o, e) => field.SetValue(o, e.Int64Value); + } + } + else if (typeof(bool?).IsAssignableFrom(memberValueType)) + { + if (property != null) + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForBool((bool?)property.GetValue(o)); + setObjectPropertyFunc = (o, e) => property.SetValue(o, e.BooleanValue); + } + else + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForBool((bool?)field.GetValue(o)); + setObjectPropertyFunc = (o, e) => field.SetValue(o, e.BooleanValue); + } + } + else if (typeof(DateTime?).IsAssignableFrom(memberValueType)) + { + if (property != null) + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForDateTimeOffset((DateTime?)property.GetValue(o)); + setObjectPropertyFunc = (o, e) => property.SetValue(o, e.DateTime); + } + else + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForDateTimeOffset((DateTime?)field.GetValue(o)); + setObjectPropertyFunc = (o, e) => field.SetValue(o, e.DateTime); + } + } + else if (typeof(DateTimeOffset?).IsAssignableFrom(memberValueType)) + { + if (property != null) + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForDateTimeOffset((DateTimeOffset?)property.GetValue(o)); + setObjectPropertyFunc = (o, e) => property.SetValue(o, e.DateTimeOffsetValue); + } + else + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForDateTimeOffset((DateTimeOffset?)field.GetValue(o)); + setObjectPropertyFunc = (o, e) => field.SetValue(o, e.DateTimeOffsetValue); + } + } + else if (typeof(Guid?).IsAssignableFrom(memberValueType)) + { + if (property != null) + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForGuid((Guid?)property.GetValue(o)); + setObjectPropertyFunc = (o, e) => property.SetValue(o, e.GuidValue); + } + else + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForGuid((Guid?)field.GetValue(o)); + setObjectPropertyFunc = (o, e) => field.SetValue(o, e.GuidValue); + } + } + else if (typeof(double?).IsAssignableFrom(memberValueType)) + { + if (property != null) + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForDouble((double?)property.GetValue(o)); + setObjectPropertyFunc = (o, e) => property.SetValue(o, e.DoubleValue); + } + else + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForDouble((double?)field.GetValue(o)); + setObjectPropertyFunc = (o, e) => field.SetValue(o, e.DoubleValue); + } + } + else if (typeof(byte[]).IsAssignableFrom(memberValueType)) + { + if (property != null) + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForByteArray((byte[])property.GetValue(o)); + setObjectPropertyFunc = (o, e) => property.SetValue(o, e.BinaryValue); + } + else + { + getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForByteArray((byte[])field.GetValue(o)); + setObjectPropertyFunc = (o, e) => field.SetValue(o, e.BinaryValue); + } + } + else // assume a serializeable object + { + getEntityPropertyFunc = o => + { + object value = property != null ? property.GetValue(o) : field.GetValue(o); + string json = value != null ? Utils.SerializeToJson(value) : null; + return EntityProperty.GeneratePropertyForString(json); + }; + + setObjectPropertyFunc = (o, e) => + { + string json = e.StringValue; + object value = json != null ? Utils.DeserializeFromJson(json, memberValueType) : null; + if (property != null) + { + property.SetValue(o, value); + } + else + { + field.SetValue(o, value); + } + }; + } + + propertyConverters.Add(new PropertyConverter(propertyName, getEntityPropertyFunc, setObjectPropertyFunc)); + } + + type = type.BaseType; + } + + return propertyConverters; + } + + class PropertyConverter + { + public PropertyConverter( + string propertyName, + Func toEntityPropertyConverter, + Action toObjectPropertyConverter) + { + this.PropertyName = propertyName; + this.GetEntityProperty = toEntityPropertyConverter; + this.SetObjectProperty = toObjectPropertyConverter; + } + + public string PropertyName { get; private set; } + public Func GetEntityProperty { get; private set; } + public Action SetObjectProperty { get; private set; } + } + } +} \ No newline at end of file diff --git a/Test/DurableTask.AzureStorage.Tests/Storage/StorageUriExtensionsTests.cs b/Test/DurableTask.AzureStorage.Tests/Storage/StorageUriExtensionsTests.cs deleted file mode 100644 index dff8128c1..000000000 --- a/Test/DurableTask.AzureStorage.Tests/Storage/StorageUriExtensionsTests.cs +++ /dev/null @@ -1,37 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- - -namespace DurableTask.AzureStorage.Tests.Storage -{ - using System; - using DurableTask.AzureStorage.Storage; - using Microsoft.VisualStudio.TestTools.UnitTesting; - using Microsoft.WindowsAzure.Storage; - - [TestClass] - public class StorageUriExtensionsTests - { - [TestMethod] - [DataRow("https://foo.blob.core.windows.net", "blob", "foo")] - [DataRow("https://bar.queue.unit.test", "queue", "bar")] - [DataRow("https://baz.table", "table", "baz")] - [DataRow("https://dev.account.file.core.windows.net", "file", "dev")] - [DataRow("https://dev.unknown.core.windows.net", "blob", null)] - [DataRow("https://host/path", "blob", null)] - [DataRow("http://127.0.0.1:10000/devstoreaccount1", "file", "devstoreaccount1")] - [DataRow("http://host:10102/devstoreaccount2/more", "blob", "devstoreaccount2")] - [DataRow("http://unit.test/custom/uri", "queue", null)] - public void GetAccountName(string uri, string service, string accountName) => - Assert.AreEqual(accountName, new StorageUri(new Uri(uri, UriKind.Absolute)).GetAccountName(service)); - } -} diff --git a/Test/DurableTask.AzureStorage.Tests/StorageAccountDetailsTests.cs b/Test/DurableTask.AzureStorage.Tests/StorageAccountDetailsTests.cs deleted file mode 100644 index 67d4709c5..000000000 --- a/Test/DurableTask.AzureStorage.Tests/StorageAccountDetailsTests.cs +++ /dev/null @@ -1,89 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- - -using System; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Auth; - -namespace DurableTask.AzureStorage.Tests -{ - [TestClass] - public class StorageAccountDetailsTests - { - [TestMethod] - public void ToCloudStorageAccount_ConnectionString() - { - var details = new StorageAccountDetails { ConnectionString = "UseDevelopmentStorage=true" }; - - CloudStorageAccount actual = details.ToCloudStorageAccount(); - Assert.AreEqual(new Uri("http://127.0.0.1:10000/devstoreaccount1", UriKind.Absolute), actual.BlobEndpoint); - Assert.AreEqual(new Uri("http://127.0.0.1:10001/devstoreaccount1", UriKind.Absolute), actual.QueueEndpoint); - Assert.AreEqual(new Uri("http://127.0.0.1:10002/devstoreaccount1", UriKind.Absolute), actual.TableEndpoint); - } - - [TestMethod] - public void ToCloudStorageAccount_Endpoints() - { - var expected = new StorageAccountDetails - { - StorageCredentials = new StorageCredentials(), - BlobServiceUri = new Uri("https://blobs", UriKind.Absolute), - QueueServiceUri = new Uri("https://queues", UriKind.Absolute), - TableServiceUri = new Uri("https://tables", UriKind.Absolute), - }; - - CloudStorageAccount actual = expected.ToCloudStorageAccount(); - Assert.AreSame(expected.StorageCredentials, actual.Credentials); - Assert.AreEqual(expected.BlobServiceUri, actual.BlobEndpoint); - Assert.AreEqual(expected.QueueServiceUri, actual.QueueEndpoint); - Assert.AreEqual(expected.TableServiceUri, actual.TableEndpoint); - } - - [DataTestMethod] - [DataRow(true, false, false)] - [DataRow(false, true, false)] - [DataRow(false, false, true)] - [DataRow(true, true, false)] - [DataRow(false, true, true)] - [DataRow(true, false, true)] - public void ToCloudStorageAccount_EndpointSubset(bool hasBlob, bool hasQueue, bool hasTable) - { - var expected = new StorageAccountDetails - { - StorageCredentials = new StorageCredentials(), - BlobServiceUri = hasBlob ? new Uri("https://blobs", UriKind.Absolute) : null, - QueueServiceUri = hasQueue ? new Uri("https://queues", UriKind.Absolute) : null, - TableServiceUri = hasTable ? new Uri("https://tables", UriKind.Absolute) : null, - }; - - Assert.ThrowsException(() => expected.ToCloudStorageAccount()); - } - - [TestMethod] - public void ToCloudStorageAccount_AccountName() - { - var expected = new StorageAccountDetails - { - AccountName = "dev", - StorageCredentials = new StorageCredentials(), - }; - - CloudStorageAccount actual = expected.ToCloudStorageAccount(); - Assert.AreSame(expected.StorageCredentials, actual.Credentials); - Assert.AreEqual(new Uri("https://dev.blob.core.windows.net", UriKind.Absolute), actual.BlobEndpoint); - Assert.AreEqual(new Uri("https://dev.queue.core.windows.net", UriKind.Absolute), actual.QueueEndpoint); - Assert.AreEqual(new Uri("https://dev.table.core.windows.net", UriKind.Absolute), actual.TableEndpoint); - } - } -} diff --git a/Test/DurableTask.AzureStorage.Tests/TableEntityConverterTests.cs b/Test/DurableTask.AzureStorage.Tests/TableEntityConverterTests.cs new file mode 100644 index 000000000..358ea8569 --- /dev/null +++ b/Test/DurableTask.AzureStorage.Tests/TableEntityConverterTests.cs @@ -0,0 +1,484 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.AzureStorage.Tests +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Runtime.Serialization; + using System.Threading.Tasks; + using Azure.Data.Tables; + using DurableTask.AzureStorage.Tests.Obsolete; + using DurableTask.AzureStorage.Tracking; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Microsoft.WindowsAzure.Storage; + + [TestClass] + public class TableEntityConverterTests + { + [TestMethod] + public void DeserializeNonNull() + { + DateTime utcNow = DateTime.UtcNow; + DateTimeOffset utcNowOffset = DateTimeOffset.UtcNow; + Guid g1 = Guid.NewGuid(); + Guid g2 = Guid.NewGuid(); + var entity = new TableEntity + { + [nameof(Example.EnumField)] = ExampleEnum.B.ToString("G"), + [nameof(Example.NullableEnumProperty)] = Utils.SerializeToJson(ExampleEnum.C), + [nameof(Example.StringProperty)] = "Hello World", + [nameof(Example.BinaryProperty)] = new byte[] { 6, 7, 8 }, + [nameof(Example.BoolProperty)] = true, + [nameof(Example.NullableBoolProperty)] = true, + ["_Timestamp"] = utcNow, + [nameof(Example.NullableDateTimeField)] = utcNow.AddDays(-1), + [nameof(Example.DateTimeOffsetProperty)] = utcNowOffset.AddYears(-5), + [nameof(Example.NullableDateTimeOffsetProperty)] = utcNowOffset.AddMonths(-2), + ["Overridden"] = 1.234D, + [nameof(Example.NullableDoubleProperty)] = 56.789D, + [nameof(Example.GuidProperty)] = g1, + [nameof(Example.NullableGuidField)] = g2, + [nameof(Example.IntField)] = 42, + [nameof(Example.NullableIntField)] = 10162022, + [nameof(Example.LongField)] = -2L, + [nameof(Example.NullableLongProperty)] = long.MaxValue, + [nameof(Example.Skipped)] = "Ignored", + [nameof(Example.UnsupportedProperty)] = Utils.SerializeToJson((short)7), + [nameof(Example.ObjectProperty)] = Utils.SerializeToJson(new Nested { Phrase = "Hello again", Number = -42 }), + }; + + Example actual = (Example)TableEntityConverter.Deserialize(entity, typeof(Example)); + + Assert.AreEqual(ExampleEnum.B, actual.EnumField); + Assert.AreEqual(ExampleEnum.C, actual.NullableEnumProperty); + Assert.AreEqual("Hello World", actual.StringProperty); + Assert.IsTrue(actual.BinaryProperty.SequenceEqual(new byte[] { 6, 7, 8 })); + Assert.IsTrue(actual.BoolProperty); + Assert.IsTrue(actual.NullableBoolProperty.Value); + Assert.AreEqual(utcNow, actual.Timestamp); + Assert.AreEqual(utcNow.AddDays(-1), actual.NullableDateTimeField); + Assert.AreEqual(utcNowOffset.AddYears(-5), actual.DateTimeOffsetProperty); + Assert.AreEqual(utcNowOffset.AddMonths(-2), actual.NullableDateTimeOffsetProperty); + Assert.AreEqual(1.234D, actual.DoubleField); + Assert.AreEqual(56.789D, actual.NullableDoubleProperty); + Assert.AreEqual(g1, actual.GuidProperty); + Assert.AreEqual(g2, actual.NullableGuidField); + Assert.AreEqual(42, actual.IntField); + Assert.AreEqual(10162022, actual.NullableIntField); + Assert.AreEqual(-2L, actual.LongField); + Assert.AreEqual(long.MaxValue, actual.NullableLongProperty); + Assert.IsNull(actual.Skipped); + Assert.AreEqual((short)7, actual.UnsupportedProperty); + Assert.AreEqual("Hello again", actual.ObjectProperty.Phrase); + Assert.AreEqual(-42, actual.ObjectProperty.Number); + } + + [TestMethod] + public void DeserializeNull() + { + // We'll both set null values and leave some values unspecified + var entity = new TableEntity + { + [nameof(Example.NullableEnumProperty)] = null, + [nameof(Example.BinaryProperty)] = null, + [nameof(Example.NullableBoolProperty)] = null, + [nameof(Example.NullableDateTimeOffsetProperty)] = null, + [nameof(Example.NullableGuidField)] = null, + [nameof(Example.NullableIntField)] = null, + [nameof(Example.ObjectProperty)] = null, + }; + + Example actual = (Example)TableEntityConverter.Deserialize(entity, typeof(Example)); + + Assert.AreEqual(ExampleEnum.A, actual.EnumField); + Assert.IsNull(actual.NullableEnumProperty); + Assert.IsNull(actual.StringProperty); + Assert.IsNull(actual.BinaryProperty); + Assert.AreEqual(default(bool), actual.BoolProperty); + Assert.IsNull(actual.NullableBoolProperty); + Assert.AreEqual(default(DateTime), actual.Timestamp); + Assert.IsNull(actual.NullableDateTimeField); + Assert.AreEqual(default(DateTimeOffset), actual.DateTimeOffsetProperty); + Assert.IsNull(actual.NullableDateTimeOffsetProperty); + Assert.AreEqual(default(double), actual.DoubleField); + Assert.IsNull(actual.NullableDoubleProperty); + Assert.AreEqual(default(Guid), actual.GuidProperty); + Assert.IsNull(actual.NullableGuidField); + Assert.AreEqual(default(int), actual.IntField); + Assert.IsNull(actual.NullableIntField); + Assert.AreEqual(default(long), actual.LongField); + Assert.IsNull(actual.NullableLongProperty); + Assert.AreEqual(default(short), actual.UnsupportedProperty); + Assert.IsNull(actual.ObjectProperty); + } + + [TestMethod] + public void SerializeNonNull() + { + var expected = new Example(default) + { + EnumField = ExampleEnum.B, + NullableEnumProperty = ExampleEnum.C, + StringProperty = "Hello World", + BinaryProperty = new byte[] { 6, 7, 8 }, + BoolProperty = true, + NullableBoolProperty = true, + Timestamp = DateTime.UtcNow, + NullableDateTimeField = DateTime.UtcNow.AddDays(-1), + DateTimeOffsetProperty = DateTimeOffset.UtcNow.AddYears(-5), + NullableDateTimeOffsetProperty = DateTimeOffset.UtcNow.AddMonths(-2), + DoubleField = 1.234, + NullableDoubleProperty = 56.789, + GuidProperty = Guid.NewGuid(), + NullableGuidField = Guid.NewGuid(), + IntField = 42, + NullableIntField = 10162022, + LongField = -2, + NullableLongProperty = long.MaxValue, + Skipped = "Not Used", + UnsupportedProperty = 7, + ObjectProperty = new Nested + { + Phrase = "Hello again", + Number = -42, + }, + }; + + AssertEntity(expected, TableEntityConverter.Serialize(expected)); + } + + [TestMethod] + public void SerializeNull() + { + // Of course, these null values are the defaults, + // but we'll set them explicitly to illustrate the purpose of the test + var expected = new Example(default) + { + NullableEnumProperty = null, + StringProperty = null, + BinaryProperty = null, + NullableBoolProperty = null, + NullableDateTimeField = null, + NullableDateTimeOffsetProperty = null, + NullableDoubleProperty = null, + NullableGuidField = null, + NullableIntField = null, + NullableLongProperty = null, + Skipped = "Not Used", + ObjectProperty = null, + }; + + AssertEntity(expected, TableEntityConverter.Serialize(expected)); + } + + [TestMethod] + public async Task BackwardsCompatible() + { + // Note: BinaryData was previously invalid in the previous converter + var oldConverter = new LegacyTableEntityConverter(); + var expected = new Example(default) + { + EnumField = ExampleEnum.B, + NullableEnumProperty = ExampleEnum.C, + StringProperty = "Hello World", + BinaryProperty = new byte[] { 6, 7, 8 }, + BoolProperty = true, + NullableBoolProperty = true, + Timestamp = DateTime.UtcNow, + NullableDateTimeField = DateTime.UtcNow.AddDays(-1), + DateTimeOffsetProperty = DateTimeOffset.UtcNow.AddYears(-5), + NullableDateTimeOffsetProperty = DateTimeOffset.UtcNow.AddMonths(-2), + DoubleField = 1.234, + NullableDoubleProperty = 56.789, + GuidProperty = Guid.NewGuid(), + NullableGuidField = Guid.NewGuid(), + IntField = 42, + NullableIntField = 10162022, + LongField = -2, + NullableLongProperty = long.MaxValue, + Skipped = "Not Used", + UnsupportedProperty = 7, + ObjectProperty = new Nested + { + Phrase = "Hello again", + Number = -42, + }, + }; + + // Create the DynamicTableEntity + var entity = oldConverter.ConvertToTableEntity(expected); + entity.PartitionKey = "12345"; + entity.RowKey = "1"; + + var legacyTableClient = CloudStorageAccount + .Parse(TestHelpers.GetTestStorageAccountConnectionString()) + .CreateCloudTableClient() + .GetTableReference(nameof(BackwardsCompatible)); + + try + { + // Initialize table and add the entity + await legacyTableClient.DeleteIfExistsAsync(); + await legacyTableClient.CreateAsync(); + await legacyTableClient.ExecuteAsync(Microsoft.WindowsAzure.Storage.Table.TableOperation.Insert(entity)); + + // Read the old entity using the new logic + var tableClient = new TableServiceClient(TestHelpers.GetTestStorageAccountConnectionString()).GetTableClient(nameof(BackwardsCompatible)); + var result = await tableClient.QueryAsync(filter: $"{nameof(ITableEntity.RowKey)} eq '1'").SingleAsync(); + + // Compare + expected.Skipped = null; + Assert.AreEqual(expected, (Example)TableEntityConverter.Deserialize(result, typeof(Example))); + } + finally + { + await legacyTableClient.DeleteIfExistsAsync(); + } + } + + [TestMethod] + public async Task ForwardsCompatible() + { + // Note: BinaryData was previously invalid in the previous converter + var expected = new Example(default) + { + EnumField = ExampleEnum.B, + NullableEnumProperty = ExampleEnum.C, + StringProperty = "Hello World", + BinaryProperty = new byte[] { 6, 7, 8 }, + BoolProperty = true, + NullableBoolProperty = true, + Timestamp = DateTime.UtcNow, + NullableDateTimeField = DateTime.UtcNow.AddDays(-1), + DateTimeOffsetProperty = DateTimeOffset.UtcNow.AddYears(-5), + NullableDateTimeOffsetProperty = DateTimeOffset.UtcNow.AddMonths(-2), + DoubleField = 1.234, + NullableDoubleProperty = 56.789, + GuidProperty = Guid.NewGuid(), + NullableGuidField = Guid.NewGuid(), + IntField = 42, + NullableIntField = 10162022, + LongField = -2, + NullableLongProperty = long.MaxValue, + Skipped = "Not Used", + UnsupportedProperty = 7, + ObjectProperty = new Nested + { + Phrase = "Hello again", + Number = -42, + }, + }; + + // Create the TableEntity + var entity = TableEntityConverter.Serialize(expected); + entity.PartitionKey = "12345"; + entity.RowKey = "1"; + + var tableClient = new TableServiceClient(TestHelpers.GetTestStorageAccountConnectionString()).GetTableClient(nameof(ForwardsCompatible)); + + try + { + // Initialize table and add the entity + await tableClient.DeleteAsync(); + await tableClient.CreateAsync(); + await tableClient.AddEntityAsync(entity); + + // Read the new entity using the old logic + var legacyTableClient = CloudStorageAccount + .Parse(TestHelpers.GetTestStorageAccountConnectionString()) + .CreateCloudTableClient() + .GetTableReference(nameof(ForwardsCompatible)); + + var segment = await legacyTableClient.ExecuteQuerySegmentedAsync( + new Microsoft.WindowsAzure.Storage.Table.TableQuery().Where( + Microsoft.WindowsAzure.Storage.Table.TableQuery.GenerateFilterCondition( + nameof(ITableEntity.RowKey), + Microsoft.WindowsAzure.Storage.Table.QueryComparisons.Equal, + "1")), + null); + + // Compare + expected.Skipped = null; + Assert.AreEqual(expected, (Example)new LegacyTableEntityConverter().ConvertFromTableEntity(segment.Single(), x => typeof(Example))); + } + finally + { + await tableClient.DeleteAsync(); + } + } + + static void AssertEntity(Example expected, TableEntity actual) + { + Assert.AreEqual(expected.EnumField.ToString(), actual.GetString(nameof(Example.EnumField))); + Assert.AreEqual(Utils.SerializeToJson(expected.NullableEnumProperty), actual.GetString(nameof(Example.NullableEnumProperty))); + Assert.AreEqual(expected.StringProperty, actual.GetString(nameof(Example.StringProperty))); + Assert.AreEqual(expected.BoolProperty, actual.GetBoolean(nameof(Example.BoolProperty))); + Assert.AreEqual(expected.NullableBoolProperty, actual.GetBoolean(nameof(Example.NullableBoolProperty))); + Assert.AreEqual(expected.Timestamp, actual.GetDateTime("_Timestamp")); + Assert.AreEqual(expected.NullableDateTimeField, actual.GetDateTime(nameof(Example.NullableDateTimeField))); + Assert.AreEqual(expected.DateTimeOffsetProperty, actual.GetDateTimeOffset(nameof(Example.DateTimeOffsetProperty))); + Assert.AreEqual(expected.NullableDateTimeOffsetProperty, actual.GetDateTimeOffset(nameof(Example.NullableDateTimeOffsetProperty))); + Assert.AreEqual(expected.DoubleField, actual.GetDouble("Overridden")); + Assert.AreEqual(expected.NullableDoubleProperty, actual.GetDouble(nameof(Example.NullableDoubleProperty))); + Assert.AreEqual(expected.GuidProperty, actual.GetGuid(nameof(Example.GuidProperty))); + Assert.AreEqual(expected.NullableGuidField, actual.GetGuid(nameof(Example.NullableGuidField))); + Assert.AreEqual(expected.IntField, actual.GetInt32(nameof(Example.IntField))); + Assert.AreEqual(expected.NullableIntField, actual.GetInt32(nameof(Example.NullableIntField))); + Assert.AreEqual(expected.LongField, actual.GetInt64(nameof(Example.LongField))); + Assert.AreEqual(expected.NullableLongProperty, actual.GetInt64(nameof(Example.NullableLongProperty))); + Assert.IsFalse(actual.ContainsKey(nameof(expected.Skipped))); + Assert.AreEqual(Utils.SerializeToJson(expected.UnsupportedProperty), actual.GetString(nameof(Example.UnsupportedProperty))); + Assert.AreEqual(Utils.SerializeToJson(expected.ObjectProperty), actual.GetString(nameof(Example.ObjectProperty))); + + if (expected.BinaryProperty == null) + { + Assert.IsNull(actual.GetBinary(nameof(Example.BinaryProperty))); + } + else + { + Assert.IsTrue(expected.BinaryProperty.SequenceEqual(actual.GetBinary(nameof(Example.BinaryProperty)))); + } + } + + [DataContract] + sealed class Example : IEquatable + { + [DataMember] + public ExampleEnum EnumField; + + [DataMember] + public ExampleEnum? NullableEnumProperty { get; set; } + + [DataMember] + public string StringProperty { get; set; } + + [DataMember] + internal byte[] BinaryProperty { get; set; } + + [DataMember] + public bool BoolProperty { get; set; } + + [DataMember] + public bool? NullableBoolProperty { get; set; } + + [DataMember] + public DateTime Timestamp { get; set; } // This will be renamed + + [DataMember] + internal DateTime? NullableDateTimeField; + + [DataMember] + public DateTimeOffset DateTimeOffsetProperty { get; set; } + + [DataMember] + public DateTimeOffset? NullableDateTimeOffsetProperty { get; set; } + + [DataMember(Name = "Overridden")] + internal double DoubleField; + + [DataMember] + internal double? NullableDoubleProperty { get; set; } + + [DataMember] + public Guid GuidProperty { get; set; } + + [DataMember] + public Guid? NullableGuidField; + + [DataMember] + public int IntField; + + [DataMember] + internal int? NullableIntField; + + [DataMember] + public long LongField; + + [DataMember] + internal long? NullableLongProperty { get; set; } + + public string Skipped { get; set; } + + [DataMember] + public short UnsupportedProperty { get; set; } + + [DataMember] + internal Nested ObjectProperty { get; set; } + + public Example(int intField) + { + this.IntField = intField; + } + + public override bool Equals(object obj) => + obj is Example other && Equals(other); + + public bool Equals(Example other) => + other != null && + EnumField == other.EnumField && + EqualityComparer.Default.Equals(NullableEnumProperty, other.NullableEnumProperty) && + StringProperty == other.StringProperty && + ArrayEquals(BinaryProperty, other.BinaryProperty) && + BoolProperty == other.BoolProperty && + EqualityComparer.Default.Equals(NullableBoolProperty, other.NullableBoolProperty) && + Timestamp == other.Timestamp && + EqualityComparer.Default.Equals(NullableDateTimeField, other.NullableDateTimeField) && + DateTimeOffsetProperty == other.DateTimeOffsetProperty && + EqualityComparer.Default.Equals(NullableDateTimeOffsetProperty, other.NullableDateTimeOffsetProperty) && + DoubleField == other.DoubleField && + EqualityComparer.Default.Equals(NullableDoubleProperty, other.NullableDoubleProperty) && + GuidProperty == other.GuidProperty && + EqualityComparer.Default.Equals(NullableGuidField, other.NullableGuidField) && + IntField == other.IntField && + EqualityComparer.Default.Equals(NullableIntField, other.NullableIntField) && + LongField == other.LongField && + EqualityComparer.Default.Equals(NullableLongProperty, other.NullableLongProperty) && + Skipped == other.Skipped && + UnsupportedProperty == other.UnsupportedProperty && + ObjectProperty.Equals(other.ObjectProperty); + + public override int GetHashCode() => + throw new NotImplementedException(); + + private static bool ArrayEquals(T[] x, T[] y) where T : IEquatable + { + if (x == null) + return y == null; + + if (y == null) + return false; + + return x.SequenceEqual(y); + } + } + + sealed class Nested : IEquatable + { + public string Phrase { get; set; } + + public int Number { get; set; } + + public bool Equals(Nested other) => + other != null && Phrase == other.Phrase && Number == other.Number; + } + + enum ExampleEnum + { + A, + B, + C, + } + } +} diff --git a/Test/DurableTask.AzureStorage.Tests/TimeoutHandlerTests.cs b/Test/DurableTask.AzureStorage.Tests/TimeoutHandlerTests.cs deleted file mode 100644 index d0d2b06e1..000000000 --- a/Test/DurableTask.AzureStorage.Tests/TimeoutHandlerTests.cs +++ /dev/null @@ -1,138 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Reflection; -using System.Text; -using System.Threading.Tasks; -using Microsoft.VisualStudio.TestTools.UnitTesting; - -namespace DurableTask.AzureStorage.Tests -{ - /// - /// Tests for . - /// - [TestClass] - public class TimeoutHandlerTests - { - /// - /// Ensures that process graceful action is executed before process is killed. - /// - /// Task tracking operation. - [TestMethod] - public async Task EnsureTimeoutHandlerRunsProcessShutdownEventsBeforeProcessKill() - { - int executionCount = 0; - int killCount = 0; - int shutdownCount = 0; - - Action killAction = (errorString) => killCount++; - typeof(TimeoutHandler) - .GetField("ProcessKillAction", BindingFlags.NonPublic | BindingFlags.Static) - .SetValue(null, killAction); - - // TimeoutHandler at the moment invokes shutdown on 5th call failure. - await TimeoutHandler.ExecuteWithTimeout( - "test", - "account", - new AzureStorageOrchestrationServiceSettings - { - OnImminentFailFast = (errorString) => - { - shutdownCount++; - return Task.FromResult(true); - } - }, - async (operationContext, cancellationToken) => - { - executionCount++; - await Task.Delay(TimeSpan.FromMinutes(3)); - return 1; - }); - - Assert.AreEqual(5, executionCount); - Assert.AreEqual(1, shutdownCount); - Assert.AreEqual(1, killCount); - } - - /// - /// Ensures that process graceful action is executed and failfast is skipped. - /// - /// Task tracking operation. - [TestMethod] - [ExpectedException(typeof(TimeoutException))] - public async Task EnsureTimeoutHandlerRunsProcessShutdownEventsAndSkipsProcessKill() - { - int executionCount = 0; - int killCount = 0; - int shutdownCount = 0; - - Action killAction = (errorString) => killCount++; - typeof(TimeoutHandler) - .GetField("ProcessKillAction", BindingFlags.NonPublic | BindingFlags.Static) - .SetValue(null, killAction); - - // TimeoutHandler at the moment invokes shutdown on 5th call failure. - await TimeoutHandler.ExecuteWithTimeout( - "test", - "account", - new AzureStorageOrchestrationServiceSettings - { - OnImminentFailFast = (errorString) => - { - shutdownCount++; - return Task.FromResult(false); - } - }, - async (operationContext, cancellationToken) => - { - executionCount++; - await Task.Delay(TimeSpan.FromMinutes(3)); - return 1; - }); - - Assert.AreEqual(5, executionCount); - Assert.AreEqual(1, shutdownCount); - Assert.AreEqual(0, killCount); - } - - /// - /// Ensures that process graceful action is executed before process is killed. - /// - /// Task tracking operation. - [TestMethod] - public async Task EnsureTimeoutHandlerExecutesProcessKillIfGracefulShutdownFails() - { - int executionCount = 0; - int killCount = 0; - int shutdownCount = 0; - - Action killAction = (errorString) => killCount++; - typeof(TimeoutHandler) - .GetField("ProcessKillAction", BindingFlags.NonPublic | BindingFlags.Static) - .SetValue(null, killAction); - - // TimeoutHandler at the moment invokes shutdown on 5th call failure. - await TimeoutHandler.ExecuteWithTimeout( - "test", - "account", - new AzureStorageOrchestrationServiceSettings - { - OnImminentFailFast = (errorString) => - { - shutdownCount++; - - throw new Exception("Breaking graceful shutdown"); - } - }, - async (operationContext, cancellationToken) => - { - executionCount++; - await Task.Delay(TimeSpan.FromMinutes(3)); - return 1; - }); - - Assert.AreEqual(5, executionCount); - Assert.AreEqual(1, shutdownCount); - Assert.AreEqual(1, killCount); - } - } -} diff --git a/samples/Correlation.Samples/TestHelpers.cs b/samples/Correlation.Samples/TestHelpers.cs index 4a228c21c..6b3eef32c 100644 --- a/samples/Correlation.Samples/TestHelpers.cs +++ b/samples/Correlation.Samples/TestHelpers.cs @@ -14,9 +14,8 @@ namespace Correlation.Samples { using System; - using DurableTask.AzureStorage; - using System.Configuration; using System.IO; + using DurableTask.AzureStorage; using Microsoft.Extensions.Configuration; public static class TestHelpers @@ -39,10 +38,10 @@ static TestHelpers() var settings = new AzureStorageOrchestrationServiceSettings { - StorageConnectionString = storageConnectionString, - TaskHubName = Configuration["taskHubName"], - ExtendedSessionsEnabled = enableExtendedSessions, ExtendedSessionIdleTimeout = TimeSpan.FromSeconds(extendedSessionTimeoutInSeconds), + ExtendedSessionsEnabled = enableExtendedSessions, + StorageAccountClientProvider = new StorageAccountClientProvider(storageConnectionString), + TaskHubName = Configuration["taskHubName"], }; return new TestOrchestrationHost(settings); diff --git a/samples/DurableTask.Samples/Program.cs b/samples/DurableTask.Samples/Program.cs index a246c760d..5bef0ed32 100644 --- a/samples/DurableTask.Samples/Program.cs +++ b/samples/DurableTask.Samples/Program.cs @@ -55,7 +55,7 @@ static void Main(string[] args) var settings = new AzureStorageOrchestrationServiceSettings { - StorageAccountDetails = new StorageAccountDetails { ConnectionString = storageConnectionString }, + StorageAccountClientProvider = new StorageAccountClientProvider(storageConnectionString), TaskHubName = taskHubName, }; diff --git a/src/DurableTask.AzureStorage/AnalyticsEventSource.cs b/src/DurableTask.AzureStorage/AnalyticsEventSource.cs index 25fc7f0a8..ababb3083 100644 --- a/src/DurableTask.AzureStorage/AnalyticsEventSource.cs +++ b/src/DurableTask.AzureStorage/AnalyticsEventSource.cs @@ -102,7 +102,7 @@ static void SetCoreTraceActivityId(Guid activityId) string ExecutionId, string MessageId, int Age, - int DequeueCount, + long DequeueCount, string NextVisibleTime, long SizeInBytes, string PartitionId, @@ -256,7 +256,7 @@ public void GeneralError(string Account, string TaskHub, string Details, string string InstanceId, string ExecutionId, string PartitionId, - int DequeueCount, + long DequeueCount, string PopReceipt, string AppName, string ExtensionVersion) @@ -287,7 +287,7 @@ public void GeneralError(string Account, string TaskHub, string Details, string string InstanceId, string ExecutionId, string PartitionId, - int DequeueCount, + long DequeueCount, string AppName, string ExtensionVersion) { diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs index 3b1993d4c..ead1a233e 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs @@ -22,6 +22,9 @@ namespace DurableTask.AzureStorage using System.Text; using System.Threading; using System.Threading.Tasks; + using Azure; + using Azure.Storage.Blobs.Models; + using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Messaging; using DurableTask.AzureStorage.Monitoring; using DurableTask.AzureStorage.Partitioning; @@ -31,7 +34,6 @@ namespace DurableTask.AzureStorage using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Query; - using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; /// @@ -40,7 +42,7 @@ namespace DurableTask.AzureStorage public sealed class AzureStorageOrchestrationService : IOrchestrationService, IOrchestrationServiceClient, - IDisposable, + IDisposable, IOrchestrationServiceQueryClient, IOrchestrationServicePurgeClient { @@ -63,7 +65,7 @@ public sealed class AzureStorageOrchestrationService : readonly ITrackingStore trackingStore; readonly ResettableLazy taskHubCreator; - readonly BlobLeaseManager leaseManager; + readonly BlobPartitionLeaseManager leaseManager; readonly AppLeaseManager appLeaseManager; readonly OrchestrationSessionManager orchestrationSessionManager; readonly IPartitionManager partitionManager; @@ -206,11 +208,11 @@ internal static string GetQueueName(string taskHub, string suffix) return queueName; } - internal static BlobLeaseManager GetBlobLeaseManager( + internal static BlobPartitionLeaseManager GetBlobLeaseManager( AzureStorageClient azureStorageClient, string leaseType) { - return new BlobLeaseManager( + return new BlobPartitionLeaseManager( azureStorageClient, leaseContainerName: azureStorageClient.Settings.TaskHubName.ToLowerInvariant() + "-leases", leaseType: leaseType); @@ -481,21 +483,21 @@ void ReportStats() this.stats.ActiveActivityExecutions.Value); } - internal async Task OnIntentLeaseAquiredAsync(BlobLease lease) + internal async Task OnIntentLeaseAquiredAsync(BlobPartitionLease lease) { var controlQueue = new ControlQueue(this.azureStorageClient, lease.PartitionId, this.messageManager); await controlQueue.CreateIfNotExistsAsync(); this.orchestrationSessionManager.ResumeListeningIfOwnQueue(lease.PartitionId, controlQueue, this.shutdownSource.Token); } - internal Task OnIntentLeaseReleasedAsync(BlobLease lease, CloseReason reason) + internal Task OnIntentLeaseReleasedAsync(BlobPartitionLease lease, CloseReason reason) { // Mark the queue as released so it will stop grabbing new messages. this.orchestrationSessionManager.ReleaseQueue(lease.PartitionId, reason, "Intent LeaseCollectionBalancer"); return Utils.CompletedTask; } - internal async Task OnOwnershipLeaseAquiredAsync(BlobLease lease) + internal async Task OnOwnershipLeaseAquiredAsync(BlobPartitionLease lease) { var controlQueue = new ControlQueue(this.azureStorageClient, lease.PartitionId, this.messageManager); await controlQueue.CreateIfNotExistsAsync(); @@ -504,16 +506,16 @@ internal async Task OnOwnershipLeaseAquiredAsync(BlobLease lease) this.allControlQueues[lease.PartitionId] = controlQueue; } - internal Task OnOwnershipLeaseReleasedAsync(BlobLease lease, CloseReason reason) + internal Task OnOwnershipLeaseReleasedAsync(BlobPartitionLease lease, CloseReason reason) { this.orchestrationSessionManager.RemoveQueue(lease.PartitionId, reason, "Ownership LeaseCollectionBalancer"); return Utils.CompletedTask; } // Used for testing - internal Task> ListBlobLeasesAsync() + internal async Task> ListBlobLeasesAsync() { - return this.partitionManager.GetOwnershipBlobLeases(); + return await this.partitionManager.GetOwnershipBlobLeases().ToListAsync(); } internal static async Task GetControlQueuesAsync( @@ -527,7 +529,7 @@ internal Task> ListBlobLeasesAsync() string taskHub = azureStorageClient.Settings.TaskHubName; - BlobLeaseManager inactiveLeaseManager = GetBlobLeaseManager(azureStorageClient, "inactive"); + BlobPartitionLeaseManager inactiveLeaseManager = GetBlobLeaseManager(azureStorageClient, "inactive"); TaskHubInfo hubInfo = await inactiveLeaseManager.GetOrCreateTaskHubInfoAsync( GetTaskHubInfo(taskHub, defaultPartitionCount), @@ -610,7 +612,7 @@ static TaskHubInfo GetTaskHubInfo(string taskHub, int partitionCount) session.ControlQueue.Name, message.TaskMessage.Event.EventType.ToString(), Utils.GetTaskEventId(message.TaskMessage.Event), - message.OriginalQueueMessage.Id, + message.OriginalQueueMessage.MessageId, message.Episode.GetValueOrDefault(-1), session.LastCheckpointTime); outOfOrderMessages.Add(message); @@ -648,7 +650,7 @@ static TaskHubInfo GetTaskHubInfo(string taskHub, int partitionCount) orchestrationWorkItem = new TaskOrchestrationWorkItem { InstanceId = session.Instance.InstanceId, - LockedUntilUtc = session.CurrentMessageBatch.Min(msg => msg.OriginalQueueMessage.NextVisibleTime.Value.UtcDateTime), + LockedUntilUtc = session.CurrentMessageBatch.Min(msg => msg.OriginalQueueMessage.NextVisibleOn.Value.UtcDateTime), NewMessages = session.CurrentMessageBatch.Select(m => m.TaskMessage).ToList(), OrchestrationRuntimeState = session.RuntimeState, Session = this.settings.ExtendedSessionsEnabled ? session : null, @@ -900,10 +902,10 @@ internal static void TraceMessageReceived(AzureStorageOrchestrationServiceSettin Utils.GetTaskEventId(taskMessage.Event), taskMessage.OrchestrationInstance.InstanceId, taskMessage.OrchestrationInstance.ExecutionId, - queueMessage.Id, - Math.Max(0, (int)DateTimeOffset.UtcNow.Subtract(queueMessage.InsertionTime.Value).TotalMilliseconds), + queueMessage.MessageId, + Math.Max(0, (int)DateTimeOffset.UtcNow.Subtract(queueMessage.InsertedOn.Value).TotalMilliseconds), queueMessage.DequeueCount, - queueMessage.NextVisibleTime.GetValueOrDefault().DateTime.ToString("o"), + queueMessage.NextVisibleOn.GetValueOrDefault().DateTime.ToString("o"), data.TotalMessageSizeBytes, data.QueueName /* PartitionId */, data.SequenceNumber, @@ -1066,26 +1068,23 @@ async Task AbandonAndReleaseSessionAsync(OrchestrationSession session) this.orchestrationSessionManager.AddMessageToPendingOrchestration(session.ControlQueue, messages, session.TraceActivityId, CancellationToken.None); } } - catch (Exception e) + catch (RequestFailedException rfe) when (rfe.Status == (int)HttpStatusCode.PreconditionFailed) { // Precondition failure is expected to be handled internally and logged as a warning. - if ((e as StorageException)?.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed) - { - // The orchestration dispatcher will handle this exception by abandoning the work item - throw new SessionAbortedException(); - } - else - { - // TODO: https://github.com/Azure/azure-functions-durable-extension/issues/332 - // It's possible that history updates may have been partially committed at this point. - // If so, what are the implications of this as far as DurableTask.Core are concerned? - this.settings.Logger.OrchestrationProcessingFailure( - this.azureStorageClient.TableAccountName, - this.settings.TaskHubName, - instanceId, - executionId, - e.ToString()); - } + // The orchestration dispatcher will handle this exception by abandoning the work item + throw new SessionAbortedException("Aborting execution due to failed precondition.", rfe); + } + catch (Exception e) + { + // TODO: https://github.com/Azure/azure-functions-durable-extension/issues/332 + // It's possible that history updates may have been partially committed at this point. + // If so, what are the implications of this as far as DurableTask.Core are concerned? + this.settings.Logger.OrchestrationProcessingFailure( + this.azureStorageClient.TableAccountName, + this.settings.TaskHubName, + instanceId, + executionId, + e.ToString()); throw; } @@ -1401,7 +1400,7 @@ async Task ReleaseSessionAsync(string instanceId) { Id = message.Id, TaskMessage = session.MessageData.TaskMessage, - LockedUntilUtc = message.OriginalQueueMessage.NextVisibleTime.Value.UtcDateTime, + LockedUntilUtc = message.OriginalQueueMessage.NextVisibleOn.Value.UtcDateTime, TraceContextBase = requestTraceContext }; @@ -1649,7 +1648,10 @@ public async Task> GetOrchestrationStateAsync(string i { // Client operations will auto-create the task hub if it doesn't already exist. await this.EnsureTaskHubAsync(); - return await this.trackingStore.GetStateAsync(instanceId, allExecutions, fetchInput: true); + return new OrchestrationState[] + { + await this.trackingStore.GetStateAsync(instanceId, allExecutions, fetchInput: true).FirstOrDefaultAsync(), + }; } /// @@ -1677,7 +1679,7 @@ public async Task> GetOrchestrationStateAsync(string i { // Client operations will auto-create the task hub if it doesn't already exist. await this.EnsureTaskHubAsync(); - return await this.trackingStore.GetStateAsync(instanceId, allExecutions, fetchInput); + return await this.trackingStore.GetStateAsync(instanceId, allExecutions, fetchInput).ToListAsync(); } /// @@ -1687,7 +1689,7 @@ public async Task> GetOrchestrationStateAsync(string i public async Task> GetOrchestrationStateAsync(CancellationToken cancellationToken = default(CancellationToken)) { await this.EnsureTaskHubAsync(); - return await this.trackingStore.GetStateAsync(cancellationToken); + return await this.trackingStore.GetStateAsync(cancellationToken).ToListAsync(); } /// @@ -1701,7 +1703,7 @@ public async Task> GetOrchestrationStateAsync(Cancella public async Task> GetOrchestrationStateAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, CancellationToken cancellationToken = default(CancellationToken)) { await this.EnsureTaskHubAsync(); - return await this.trackingStore.GetStateAsync(createdTimeFrom, createdTimeTo, runtimeStatus, cancellationToken); + return await this.trackingStore.GetStateAsync(createdTimeFrom, createdTimeTo, runtimeStatus, cancellationToken).ToListAsync(); } /// @@ -1717,7 +1719,14 @@ public async Task> GetOrchestrationStateAsync(DateTime public async Task GetOrchestrationStateAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, int top, string continuationToken, CancellationToken cancellationToken = default(CancellationToken)) { await this.EnsureTaskHubAsync(); - return await this.trackingStore.GetStateAsync(createdTimeFrom, createdTimeTo, runtimeStatus, top, continuationToken, cancellationToken); + Page page = await this.trackingStore + .GetStateAsync(createdTimeFrom, createdTimeTo, runtimeStatus, cancellationToken) + .AsPages(continuationToken, top) + .FirstOrDefaultAsync(); + + return page != null + ? new DurableStatusQueryResult { ContinuationToken = page.ContinuationToken, OrchestrationState = page.Values } + : new DurableStatusQueryResult { OrchestrationState = Array.Empty() }; } /// @@ -1731,7 +1740,14 @@ public async Task GetOrchestrationStateAsync(DateTime public async Task GetOrchestrationStateAsync(OrchestrationInstanceStatusQueryCondition condition, int top, string continuationToken, CancellationToken cancellationToken = default(CancellationToken)) { await this.EnsureTaskHubAsync(); - return await this.trackingStore.GetStateAsync(condition, top, continuationToken, cancellationToken); + Page page = await this.trackingStore + .GetStateAsync(condition, cancellationToken) + .AsPages(continuationToken, top) + .FirstOrDefaultAsync(); + + return page != null + ? new DurableStatusQueryResult { ContinuationToken = page.ContinuationToken, OrchestrationState = page.Values } + : new DurableStatusQueryResult { OrchestrationState = Array.Empty() }; } /// @@ -1757,7 +1773,7 @@ public Task ForceTerminateTaskOrchestrationAsync(string instanceId, string reaso /// The reason for rewinding. public async Task RewindTaskOrchestrationAsync(string instanceId, string reason) { - var queueIds = await this.trackingStore.RewindHistoryAsync(instanceId, new List(), default(CancellationToken)); + List queueIds = await this.trackingStore.RewindHistoryAsync(instanceId).ToListAsync(); foreach (string id in queueIds) { @@ -1979,16 +1995,6 @@ private static OrchestrationQueryResult ConvertFrom(DurableStatusQueryResult sta return new OrchestrationQueryResult(results, statusContext.ContinuationToken); } - class PendingMessageBatch - { - public string OrchestrationInstanceId { get; set; } - public string OrchestrationExecutionId { get; set; } - - public List Messages { get; set; } = new List(); - - public OrchestrationRuntimeState Orchestrationstate { get; set; } - } - class ResettableLazy { readonly Func valueFactory; diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs index c84621951..e29a3f0da 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs @@ -10,18 +10,16 @@ // See the License for the specific language governing permissions and // limitations under the License. // ---------------------------------------------------------------------------------- - +#nullable enable namespace DurableTask.AzureStorage { using System; - using DurableTask.AzureStorage.Partitioning; + using System.Runtime.Serialization; + using Azure.Data.Tables; using DurableTask.AzureStorage.Logging; + using DurableTask.AzureStorage.Partitioning; using DurableTask.Core; using Microsoft.Extensions.Logging; - using Microsoft.WindowsAzure.Storage.Queue; - using Microsoft.WindowsAzure.Storage.Table; - using System.Runtime.Serialization; - using System.Threading.Tasks; /// /// Settings that impact the runtime behavior of the . @@ -32,7 +30,7 @@ public class AzureStorageOrchestrationServiceSettings internal static readonly TimeSpan DefaultMaxQueuePollingInterval = TimeSpan.FromSeconds(30); - LogHelper logHelper; + LogHelper? logHelper; /// /// Gets or sets the name of the app. @@ -56,42 +54,18 @@ public class AzureStorageOrchestrationServiceSettings /// public TimeSpan ControlQueueVisibilityTimeout { get; set; } = TimeSpan.FromMinutes(5); - /// - /// Gets or sets the that are provided to all internal - /// usage of APIs for the control queue. - /// - [Obsolete("ControlQueueRequestOptions is deprecated. If you still need to configure QueueRequestOptions please open an issue at https://github.com/Azure/durabletask with the specific configurations options you need.")] - public QueueRequestOptions ControlQueueRequestOptions { get; set; } - /// /// Gets or sets the visibility timeout of dequeued work item queue messages. The default is 5 minutes. /// public TimeSpan WorkItemQueueVisibilityTimeout { get; set; } = TimeSpan.FromMinutes(5); - /// - /// Gets or sets the that are provided to all internal - /// usage of APIs for the work item queue. - /// - [Obsolete("WorkItemQueueRequestOptions is deprecated. If you still need to configure QueueRequestOptions please open an issue at https://github.com/Azure/durabletask with the specific configurations options you need.")] - public QueueRequestOptions WorkItemQueueRequestOptions { get; set; } - - /// - /// Gets or sets the that are provided to all internal - /// usage of the APIs for the history table. - /// - [Obsolete("HistoryTableRequestOptions is deprecated. If you still need to configure TableRequestOptions please open an issue at https://github.com/Azure/durabletask with the specific configurations options you need.")] - public TableRequestOptions HistoryTableRequestOptions { get; set; } - - /// - /// Gets or sets the Azure Storage connection string. - /// - public string StorageConnectionString { get; set; } - /// /// Gets or sets the prefix of the TrackingStore table name. - /// This property is only used when we have TrackingStoreStorageAccountDetails. - /// The default is "DurableTask" /// + /// + /// This property is only used when the value of the property + /// is not . The default is "DurableTask". + /// public string TrackingStoreNamePrefix { get; set; } = "DurableTask"; /// @@ -181,17 +155,19 @@ public class AzureStorageOrchestrationServiceSettings public AppLeaseOptions AppLeaseOptions { get; set; } = AppLeaseOptions.DefaultOptions; /// - /// Gets or sets the Azure Storage Account details - /// If provided, this is used to connect to Azure Storage + /// Gets or sets the client provider for the Azure Storage Account services used by the Durable Task Framework. /// - public StorageAccountDetails StorageAccountDetails { get; set; } + public StorageAccountClientProvider? StorageAccountClientProvider { get; set; } /// - /// Gets or sets the Storage Account Details for Tracking Store. - /// In case of null, StorageAccountDetails is applied. + /// Gets or sets the storage provider for Azure Table Storage, which is used for the Tracking Store + /// that records the progress of orchestrations and entities. /// - public StorageAccountDetails TrackingStoreStorageAccountDetails { get; set; } - + /// + /// If the value is , then the is used instead. + /// + public IStorageServiceClientProvider? TrackingServiceClientProvider { get; set; } + /// /// Should we carry over unexecuted raised events to the next iteration of an orchestration on ContinueAsNew /// @@ -224,36 +200,9 @@ public class AzureStorageOrchestrationServiceSettings public bool DisableExecutionStartedDeduplication { get; set; } /// - /// Gets or sets an optional function to be executed before the app is recycled. Reason for shutdown is passed as a string parameter. - /// This can be used to perform any pending cleanup tasks or just do a graceful shutdown. - /// The function returns a . If 'true' is returned is executed, if 'false' is returned, - /// process kill is skipped. - /// A wait time of 35 seconds will be given for the task to finish, if the task does not finish in required time, will be executed. - /// - /// Skipping process kill by returning false might have negative consequences if since Storage SDK might be in deadlock. Ensure if you return - /// false a process shutdown is executed by you. - public Func> OnImminentFailFast { get; set; } = (message) => Task.FromResult(true); - - /// - /// Gets or sets the number of times we allow the timeout to be hit before recycling the app. We set this - /// to a fixed value to prevent building up an infinite number of deadlocked tasks and leak resources. - /// - public int MaxNumberOfTimeoutsBeforeRecycle { get; set; } = 5; - - /// - /// Gets or sets the number of seconds before a request to Azure Storage is considered as timed out. - /// - public TimeSpan StorageRequestsTimeout { get; set; } = TimeSpan.FromMinutes(2); - - /// - /// Gets or sets the window duration (in seconds) in which we count the number of timed out request before recycling the app. - /// - public TimeSpan StorageRequestsTimeoutCooldown { get; set; } = TimeSpan.FromMinutes(5); - - /// - /// Returns bool indicating is the TrackingStoreStorageAccount has been set. + /// Returns bool indicating is the has been set. /// - public bool HasTrackingStoreStorageAccount => this.TrackingStoreStorageAccountDetails != null; + public bool HasTrackingStoreStorageAccount => this.TrackingServiceClientProvider != null; internal string HistoryTableName => this.HasTrackingStoreStorageAccount ? $"{this.TrackingStoreNamePrefix}History" : $"{this.TaskHubName}History"; diff --git a/src/DurableTask.AzureStorage/DefaultStorageServiceClientProvider.cs b/src/DurableTask.AzureStorage/DefaultStorageServiceClientProvider.cs new file mode 100644 index 000000000..90a27295c --- /dev/null +++ b/src/DurableTask.AzureStorage/DefaultStorageServiceClientProvider.cs @@ -0,0 +1,40 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage +{ + using System; + using Azure.Core; + + sealed class DefaultStorageServiceClientProvider : IStorageServiceClientProvider where TClientOptions : ClientOptions + { + readonly Func factory; + readonly TClientOptions options; + + public DefaultStorageServiceClientProvider(Func factory, TClientOptions options) + { + this.factory = factory ?? throw new ArgumentNullException(nameof(factory)); + this.options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public TClient CreateClient(TClientOptions options) + { + return this.factory(options); + } + + public TClientOptions CreateOptions() + { + return this.options; + } + } +} diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 13969a527..2e9e80eed 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -19,11 +19,12 @@ - 1 - 13 - 3 + 2 + 0 + 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) + preview.1 $(VersionPrefix).0 $(VersionPrefix).$(FileVersionRevision) @@ -33,17 +34,24 @@ $(VersionPrefix) + + + + + + + + + + - - - diff --git a/src/DurableTask.AzureStorage/Http/LeaseTimeoutHttpPipelinePolicy.cs b/src/DurableTask.AzureStorage/Http/LeaseTimeoutHttpPipelinePolicy.cs new file mode 100644 index 000000000..1bdecd0cc --- /dev/null +++ b/src/DurableTask.AzureStorage/Http/LeaseTimeoutHttpPipelinePolicy.cs @@ -0,0 +1,55 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Http +{ + using System; + using System.Threading.Tasks; + using Azure.Core; + using Azure.Core.Pipeline; + + sealed class LeaseTimeoutHttpPipelinePolicy : HttpPipelinePolicy + { + readonly TimeSpan leaseRenewalTimeout; + + public LeaseTimeoutHttpPipelinePolicy(TimeSpan leaseRenewalTimeout) + { + this.leaseRenewalTimeout = leaseRenewalTimeout; + } + + public override void Process(HttpMessage message, ReadOnlyMemory pipeline) + { + if (IsBlobLeaseRenewal(message.Request.Uri.Query, message.Request.Headers)) + { + message.NetworkTimeout = this.leaseRenewalTimeout; + } + + ProcessNext(message, pipeline); + } + + public override ValueTask ProcessAsync(HttpMessage message, ReadOnlyMemory pipeline) + { + if (IsBlobLeaseRenewal(message.Request.Uri.Query, message.Request.Headers)) + { + message.NetworkTimeout = this.leaseRenewalTimeout; + } + + return ProcessNextAsync(message, pipeline); + } + + static bool IsBlobLeaseRenewal(string query, RequestHeaders headers) => + query.IndexOf("comp=lease", StringComparison.OrdinalIgnoreCase) != -1 && + headers.TryGetValue("x-ms-lease-action", out string? value) && + string.Equals(value, "renew", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/DurableTask.AzureStorage/Http/MonitoringHttpPipelinePolicy.cs b/src/DurableTask.AzureStorage/Http/MonitoringHttpPipelinePolicy.cs new file mode 100644 index 000000000..bdc26cf8d --- /dev/null +++ b/src/DurableTask.AzureStorage/Http/MonitoringHttpPipelinePolicy.cs @@ -0,0 +1,43 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Http +{ + using System; + using System.Threading.Tasks; + using Azure.Core; + using Azure.Core.Pipeline; + using DurableTask.AzureStorage.Monitoring; + + sealed class MonitoringHttpPipelinePolicy : HttpPipelinePolicy + { + readonly AzureStorageOrchestrationServiceStats stats; + + public MonitoringHttpPipelinePolicy(AzureStorageOrchestrationServiceStats stats) + { + this.stats = stats ?? throw new ArgumentNullException(nameof(stats)); + } + + public override void Process(HttpMessage message, ReadOnlyMemory pipeline) + { + this.stats.StorageRequests.Increment(); + ProcessNext(message, pipeline); + } + + public override ValueTask ProcessAsync(HttpMessage message, ReadOnlyMemory pipeline) + { + this.stats.StorageRequests.Increment(); + return ProcessNextAsync(message, pipeline); + } + } +} diff --git a/src/DurableTask.AzureStorage/Http/ThrottlingHttpPipelinePolicy.cs b/src/DurableTask.AzureStorage/Http/ThrottlingHttpPipelinePolicy.cs new file mode 100644 index 000000000..eb1662566 --- /dev/null +++ b/src/DurableTask.AzureStorage/Http/ThrottlingHttpPipelinePolicy.cs @@ -0,0 +1,75 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Http +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Azure.Core; + using Azure.Core.Pipeline; + + sealed class ThrottlingHttpPipelinePolicy : HttpPipelinePolicy, IDisposable + { + readonly SemaphoreSlim throttle; + + public ThrottlingHttpPipelinePolicy(int maxRequests) + { + this.throttle = new SemaphoreSlim(maxRequests); + } + + public void Dispose() + { + this.throttle.Dispose(); + } + + public override void Process(HttpMessage message, ReadOnlyMemory pipeline) + { + if (IsBlobLease(message.Request.Uri.Query)) + { + ProcessNext(message, pipeline); + } + else + { + this.throttle.Wait(message.CancellationToken); + try + { + ProcessNext(message, pipeline); + } + finally + { + this.throttle.Release(); + } + } + } + + public override ValueTask ProcessAsync(HttpMessage message, ReadOnlyMemory pipeline) => + IsBlobLease(message.Request.Uri.Query) ? ProcessNextAsync(message, pipeline) : this.ThrottledProcessNextAsync(message, pipeline); + + async ValueTask ThrottledProcessNextAsync(HttpMessage message, ReadOnlyMemory pipeline) + { + await this.throttle.WaitAsync(message.CancellationToken); + try + { + await ProcessNextAsync(message, pipeline); + } + finally + { + this.throttle.Release(); + } + } + + static bool IsBlobLease(string query) => + query.IndexOf("comp=lease", StringComparison.OrdinalIgnoreCase) != -1; + } +} diff --git a/src/DurableTask.AzureStorage/IStorageServiceClientProvider.cs b/src/DurableTask.AzureStorage/IStorageServiceClientProvider.cs new file mode 100644 index 000000000..821ad0a7b --- /dev/null +++ b/src/DurableTask.AzureStorage/IStorageServiceClientProvider.cs @@ -0,0 +1,41 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage +{ + using Azure.Core; + + /// + /// Represents a factory for a particular Azure Storage service client. + /// + /// The type of the client. + /// The type of the options used by the client. + public interface IStorageServiceClientProvider where TClientOptions : ClientOptions + { + /// + /// Creates the options for the client. + /// + /// + /// The result may be modified by callers before invoking . + /// + /// The corresponding client options. + TClientOptions CreateOptions(); + + /// + /// Creates the client based on the given . + /// + /// Options for the client. + /// The corresponding client. + TClient CreateClient(TClientOptions options); + } +} diff --git a/src/DurableTask.AzureStorage/Logging/LogEvents.cs b/src/DurableTask.AzureStorage/Logging/LogEvents.cs index d7e4af427..721577ab6 100644 --- a/src/DurableTask.AzureStorage/Logging/LogEvents.cs +++ b/src/DurableTask.AzureStorage/Logging/LogEvents.cs @@ -133,7 +133,7 @@ internal class ReceivedMessage : StructuredLogEvent, IEventSourceEvent string executionId, string messageId, int age, - int dequeueCount, + long dequeueCount, string nextVisibleTime, long sizeInBytes, string partitionId, @@ -186,7 +186,7 @@ internal class ReceivedMessage : StructuredLogEvent, IEventSourceEvent public int Age { get; } [StructuredLogField] - public int DequeueCount { get; } + public long DequeueCount { get; } [StructuredLogField] public string NextVisibleTime { get; } @@ -581,7 +581,7 @@ internal class DuplicateMessageDetected : StructuredLogEvent, IEventSourceEvent string instanceId, string executionId, string partitionId, - int dequeueCount, + long dequeueCount, string popReceipt) { this.Account = account; @@ -621,7 +621,7 @@ internal class DuplicateMessageDetected : StructuredLogEvent, IEventSourceEvent public string PartitionId { get; } [StructuredLogField] - public int DequeueCount { get; } + public long DequeueCount { get; } [StructuredLogField] public string PopReceipt { get; } @@ -666,7 +666,7 @@ internal class PoisonMessageDetected : StructuredLogEvent, IEventSourceEvent string instanceId, string executionId, string partitionId, - int dequeueCount) + long dequeueCount) { this.Account = account; this.TaskHub = taskHub; @@ -704,7 +704,7 @@ internal class PoisonMessageDetected : StructuredLogEvent, IEventSourceEvent public string PartitionId { get; } [StructuredLogField] - public int DequeueCount { get; } + public long DequeueCount { get; } public override EventId EventId => new EventId( EventIds.PoisonMessageDetected, diff --git a/src/DurableTask.AzureStorage/Logging/LogHelper.cs b/src/DurableTask.AzureStorage/Logging/LogHelper.cs index 929fa20f9..104615ada 100644 --- a/src/DurableTask.AzureStorage/Logging/LogHelper.cs +++ b/src/DurableTask.AzureStorage/Logging/LogHelper.cs @@ -69,7 +69,7 @@ public LogHelper(ILogger log) string executionId, string messageId, int age, - int dequeueCount, + long dequeueCount, string nextVisibleTime, long sizeInBytes, string partitionId, @@ -210,7 +210,7 @@ public LogHelper(ILogger log) string instanceId, string executionId, string partitionId, - int dequeueCount, + long dequeueCount, string popReceipt) { var logEvent = new LogEvents.DuplicateMessageDetected( @@ -236,7 +236,7 @@ public LogHelper(ILogger log) string instanceId, string executionId, string partitionId, - int dequeueCount) + long dequeueCount) { var logEvent = new LogEvents.PoisonMessageDetected( account, diff --git a/src/DurableTask.AzureStorage/MessageData.cs b/src/DurableTask.AzureStorage/MessageData.cs index d0fb93594..bf39fbc72 100644 --- a/src/DurableTask.AzureStorage/MessageData.cs +++ b/src/DurableTask.AzureStorage/MessageData.cs @@ -15,6 +15,7 @@ namespace DurableTask.AzureStorage { using System; using System.Runtime.Serialization; + using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Storage; using DurableTask.Core; using Newtonsoft.Json; @@ -96,7 +97,7 @@ public MessageData() [DataMember] public string SerializableTraceContext { get; set; } - internal string Id => this.OriginalQueueMessage?.Id; + internal string Id => this.OriginalQueueMessage?.MessageId; internal string QueueName { get; set; } diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index 9f77fbb00..c97bc26a9 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -14,16 +14,23 @@ namespace DurableTask.AzureStorage { using System; - using System.Collections.Generic; using System.IO; using System.IO.Compression; + using System.Linq; using System.Reflection; +#if !NETSTANDARD2_0 using System.Runtime.Serialization; +#endif using System.Text; + using System.Threading; using System.Threading.Tasks; + using Azure; + using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Storage; using Newtonsoft.Json; +#if NETSTANDARD2_0 using Newtonsoft.Json.Serialization; +#endif /// /// The message manager for messages from MessageData, and DynamicTableEntities @@ -68,27 +75,27 @@ class MessageManager } } - public async Task EnsureContainerAsync() + public async Task EnsureContainerAsync(CancellationToken cancellationToken = default) { bool created = false; if (!this.containerInitialized) { - created = await this.blobContainer.CreateIfNotExistsAsync(); + created = await this.blobContainer.CreateIfNotExistsAsync(cancellationToken); this.containerInitialized = true; } return created; } - public async Task DeleteContainerAsync() + public async Task DeleteContainerAsync(CancellationToken cancellationToken = default) { - bool deleted = await this.blobContainer.DeleteIfExistsAsync(); + bool deleted = await this.blobContainer.DeleteIfExistsAsync(cancellationToken: cancellationToken); this.containerInitialized = false; return deleted; } - public async Task SerializeMessageDataAsync(MessageData messageData) + public async Task SerializeMessageDataAsync(MessageData messageData, CancellationToken cancellationToken = default) { string rawContent = Utils.SerializeToJson(serializer, messageData); messageData.TotalMessageSizeBytes = Encoding.UTF8.GetByteCount(rawContent); @@ -100,7 +107,7 @@ public async Task SerializeMessageDataAsync(MessageData messageData) byte[] messageBytes = Encoding.UTF8.GetBytes(rawContent); string blobName = this.GetNewLargeMessageBlobName(messageData); messageData.CompressedBlobName = blobName; - await this.CompressAndUploadAsBytesAsync(messageBytes, blobName); + await this.CompressAndUploadAsBytesAsync(messageBytes, blobName, cancellationToken); // Create a "wrapper" message which has the blob name but not a task message. var wrapperMessageData = new MessageData { CompressedBlobName = blobName }; @@ -115,12 +122,13 @@ public async Task SerializeMessageDataAsync(MessageData messageData) /// Otherwise returns the message as is. /// /// The message to be fetched if it is a url. + /// A token used for canceling the operation. /// Actual string representation of message. - public async Task FetchLargeMessageIfNecessary(string message) + public async Task FetchLargeMessageIfNecessary(string message, CancellationToken cancellationToken = default) { if (TryGetLargeMessageReference(message, out Uri blobUrl)) { - return await this.DownloadAndDecompressAsBytesAsync(blobUrl); + return await this.DownloadAndDecompressAsBytesAsync(blobUrl, cancellationToken); } else { @@ -133,20 +141,22 @@ internal static bool TryGetLargeMessageReference(string messagePayload, out Uri return Uri.TryCreate(messagePayload, UriKind.Absolute, out blobUrl); } - public async Task DeserializeQueueMessageAsync(QueueMessage queueMessage, string queueName) + public async Task DeserializeQueueMessageAsync(QueueMessage queueMessage, string queueName, CancellationToken cancellationToken = default) { - MessageData envelope = this.DeserializeMessageData(queueMessage.Message); + // TODO: Deserialize with Stream? + byte[] body = queueMessage.Body.ToArray(); + MessageData envelope = this.DeserializeMessageData(Encoding.UTF8.GetString(body)); if (!string.IsNullOrEmpty(envelope.CompressedBlobName)) { - string decompressedMessage = await this.DownloadAndDecompressAsBytesAsync(envelope.CompressedBlobName); + string decompressedMessage = await this.DownloadAndDecompressAsBytesAsync(envelope.CompressedBlobName, cancellationToken); envelope = this.DeserializeMessageData(decompressedMessage); envelope.MessageFormat = MessageFormatFlags.StorageBlob; envelope.TotalMessageSizeBytes = Encoding.UTF8.GetByteCount(decompressedMessage); } else { - envelope.TotalMessageSizeBytes = Encoding.UTF8.GetByteCount(queueMessage.Message); + envelope.TotalMessageSizeBytes = body.Length; } envelope.OriginalQueueMessage = queueMessage; @@ -159,13 +169,12 @@ internal MessageData DeserializeMessageData(string json) return Utils.DeserializeFromJson(this.serializer, json); } - public Task CompressAndUploadAsBytesAsync(byte[] payloadBuffer, string blobName) + public Task CompressAndUploadAsBytesAsync(byte[] payloadBuffer, string blobName, CancellationToken cancellationToken = default) { ArraySegment compressedSegment = this.Compress(payloadBuffer); - return this.UploadToBlobAsync(compressedSegment.Array, compressedSegment.Count, blobName); + return this.UploadToBlobAsync(compressedSegment.Array, compressedSegment.Count, blobName, cancellationToken); } - [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2202:DoNotDisposeObjectsMultipleTimes", Justification = "This GZipStream will not dispose the MemoryStream.")] public ArraySegment Compress(byte[] payloadBuffer) { using (var originStream = new MemoryStream(payloadBuffer, 0, payloadBuffer.Length)) @@ -196,25 +205,25 @@ public ArraySegment Compress(byte[] payloadBuffer) } } - public async Task DownloadAndDecompressAsBytesAsync(string blobName) + public async Task DownloadAndDecompressAsBytesAsync(string blobName, CancellationToken cancellationToken = default) { - await this.EnsureContainerAsync(); + await this.EnsureContainerAsync(cancellationToken); Blob blob = this.blobContainer.GetBlobReference(blobName); - return await DownloadAndDecompressAsBytesAsync(blob); + return await DownloadAndDecompressAsBytesAsync(blob, cancellationToken); } - public Task DownloadAndDecompressAsBytesAsync(Uri blobUri) + public Task DownloadAndDecompressAsBytesAsync(Uri blobUri, CancellationToken cancellationToken = default) { Blob blob = this.azureStorageClient.GetBlobReference(blobUri); - return DownloadAndDecompressAsBytesAsync(blob); + return DownloadAndDecompressAsBytesAsync(blob, cancellationToken); } - private async Task DownloadAndDecompressAsBytesAsync(Blob blob) + private async Task DownloadAndDecompressAsBytesAsync(Blob blob, CancellationToken cancellationToken = default) { using (MemoryStream memory = new MemoryStream(MaxStorageQueuePayloadSizeInBytes * 2)) { - await blob.DownloadToStreamAsync(memory); + await blob.DownloadToStreamAsync(memory, cancellationToken); memory.Position = 0; ArraySegment decompressedSegment = this.Decompress(memory); @@ -264,12 +273,12 @@ public MessageFormatFlags GetMessageFormatFlags(MessageData messageData) return messageFormatFlags; } - public async Task UploadToBlobAsync(byte[] data, int dataByteCount, string blobName) + public async Task UploadToBlobAsync(byte[] data, int dataByteCount, string blobName, CancellationToken cancellationToken = default) { - await this.EnsureContainerAsync(); + await this.EnsureContainerAsync(cancellationToken); Blob blob = this.blobContainer.GetBlobReference(blobName); - await blob.UploadFromByteArrayAsync(data, 0, dataByteCount); + await blob.UploadFromByteArrayAsync(data, 0, dataByteCount, cancellationToken); } public string GetNewLargeMessageBlobName(MessageData message) @@ -281,22 +290,19 @@ public string GetNewLargeMessageBlobName(MessageData message) return $"{instanceId}/message-{activityId}-{eventType}.json.gz"; } - public async Task DeleteLargeMessageBlobs(string sanitizedInstanceId) + public async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, CancellationToken cancellationToken = default) { int storageOperationCount = 1; - if (await this.blobContainer.ExistsAsync()) + if (await this.blobContainer.ExistsAsync(cancellationToken)) { - IEnumerable blobList = await this.blobContainer.ListBlobsAsync(sanitizedInstanceId); - storageOperationCount++; - - var blobForDeletionTaskList = new List(); - foreach (Blob blob in blobList) + await foreach (Page page in this.blobContainer.ListBlobsAsync(sanitizedInstanceId, cancellationToken).AsPages()) { - blobForDeletionTaskList.Add(blob.DeleteIfExistsAsync()); - } + storageOperationCount++; + + await Task.WhenAll(page.Values.Select(b => b.DeleteIfExistsAsync(cancellationToken))); - await Task.WhenAll(blobForDeletionTaskList); - storageOperationCount += blobForDeletionTaskList.Count; + storageOperationCount += page.Values.Count; + } } return storageOperationCount; diff --git a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs index 59024c12f..68e924a50 100644 --- a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs +++ b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs @@ -19,6 +19,7 @@ namespace DurableTask.AzureStorage.Messaging using System.Linq; using System.Threading; using System.Threading.Tasks; + using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Monitoring; using DurableTask.AzureStorage.Partitioning; using DurableTask.AzureStorage.Storage; @@ -116,7 +117,7 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) this.settings.Logger.MessageFailure( this.storageAccountName, this.settings.TaskHubName, - queueMessage.Id /* MessageId */, + queueMessage.MessageId /* MessageId */, string.Empty /* InstanceId */, string.Empty /* ExecutionId */, this.storageQueue.Name, @@ -130,7 +131,7 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) } // Check to see whether we've already dequeued this message. - if (!this.stats.PendingOrchestratorMessages.TryAdd(queueMessage.Id, 1)) + if (!this.stats.PendingOrchestratorMessages.TryAdd(queueMessage.MessageId, 1)) { // This message is already loaded in memory and is therefore a duplicate. // We will continue to process it because we need the updated pop receipt. @@ -139,7 +140,7 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) this.settings.TaskHubName, messageData.TaskMessage.Event.EventType.ToString(), Utils.GetTaskEventId(messageData.TaskMessage.Event), - queueMessage.Id, + queueMessage.MessageId, messageData.TaskMessage.OrchestrationInstance.InstanceId, messageData.TaskMessage.OrchestrationInstance.ExecutionId, this.Name, @@ -192,7 +193,7 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) // This overload is intended for cases where we aren't able to deserialize an instance of MessageData. public Task AbandonMessageAsync(QueueMessage queueMessage) { - this.stats.PendingOrchestratorMessages.TryRemove(queueMessage.Id, out _); + this.stats.PendingOrchestratorMessages.TryRemove(queueMessage.MessageId, out _); return base.AbandonMessageAsync( queueMessage, taskMessage: null, @@ -203,13 +204,13 @@ public Task AbandonMessageAsync(QueueMessage queueMessage) public override Task AbandonMessageAsync(MessageData message, SessionBase? session = null) { - this.stats.PendingOrchestratorMessages.TryRemove(message.OriginalQueueMessage.Id, out _); + this.stats.PendingOrchestratorMessages.TryRemove(message.OriginalQueueMessage.MessageId, out _); return base.AbandonMessageAsync(message, session); } public override Task DeleteMessageAsync(MessageData message, SessionBase? session = null) { - this.stats.PendingOrchestratorMessages.TryRemove(message.OriginalQueueMessage.Id, out _); + this.stats.PendingOrchestratorMessages.TryRemove(message.OriginalQueueMessage.MessageId, out _); return base.DeleteMessageAsync(message, session); } @@ -243,8 +244,8 @@ public int Compare(MessageData x, MessageData y) { // Azure Storage is the ultimate authority on the order in which messages were received. // Insertion time only has full second precision, however, so it's not always useful. - DateTimeOffset insertionTimeX = x.OriginalQueueMessage.InsertionTime.GetValueOrDefault(); - DateTimeOffset insertionTimeY = y.OriginalQueueMessage.InsertionTime.GetValueOrDefault(); + DateTimeOffset insertionTimeX = x.OriginalQueueMessage.InsertedOn.GetValueOrDefault(); + DateTimeOffset insertionTimeY = y.OriginalQueueMessage.InsertedOn.GetValueOrDefault(); if (insertionTimeX != insertionTimeY) { return insertionTimeX.CompareTo(insertionTimeY); diff --git a/src/DurableTask.AzureStorage/Messaging/MessageCollection.cs b/src/DurableTask.AzureStorage/Messaging/MessageCollection.cs index b23df39d1..024ae74ab 100644 --- a/src/DurableTask.AzureStorage/Messaging/MessageCollection.cs +++ b/src/DurableTask.AzureStorage/Messaging/MessageCollection.cs @@ -14,7 +14,7 @@ namespace DurableTask.AzureStorage.Messaging { using System.Collections.Generic; - using DurableTask.AzureStorage.Storage; + using Azure.Storage.Queues.Models; class MessageCollection : List { @@ -29,7 +29,7 @@ public void AddOrReplace(MessageData message) for (int i = 0; i < this.Count; i++) { QueueMessage existingMessage = this[i].OriginalQueueMessage; - if (existingMessage.Id == message.OriginalQueueMessage.Id) + if (existingMessage.MessageId == message.OriginalQueueMessage.MessageId) { this[i] = message; return; diff --git a/src/DurableTask.AzureStorage/Messaging/OrchestrationSession.cs b/src/DurableTask.AzureStorage/Messaging/OrchestrationSession.cs index 96a6318ff..98acdf885 100644 --- a/src/DurableTask.AzureStorage/Messaging/OrchestrationSession.cs +++ b/src/DurableTask.AzureStorage/Messaging/OrchestrationSession.cs @@ -18,6 +18,7 @@ namespace DurableTask.AzureStorage.Messaging using System.IO; using System.Linq; using System.Threading.Tasks; + using Azure; using DurableTask.Core; using DurableTask.Core.History; using Newtonsoft.Json; @@ -36,7 +37,7 @@ sealed class OrchestrationSession : SessionBase, IOrchestrationSession ControlQueue controlQueue, List initialMessageBatch, OrchestrationRuntimeState runtimeState, - string eTag, + ETag? eTag, DateTime lastCheckpointTime, TimeSpan idleTimeout, Guid traceActivityId) @@ -63,7 +64,7 @@ sealed class OrchestrationSession : SessionBase, IOrchestrationSession public OrchestrationRuntimeState RuntimeState { get; private set; } - public string ETag { get; set; } + public ETag? ETag { get; set; } public DateTime LastCheckpointTime { get; } diff --git a/src/DurableTask.AzureStorage/Messaging/Session.cs b/src/DurableTask.AzureStorage/Messaging/Session.cs index 9153a7aa6..b0771740f 100644 --- a/src/DurableTask.AzureStorage/Messaging/Session.cs +++ b/src/DurableTask.AzureStorage/Messaging/Session.cs @@ -14,7 +14,7 @@ namespace DurableTask.AzureStorage.Messaging { using System; - using DurableTask.AzureStorage.Storage; + using Azure.Storage.Queues.Models; using DurableTask.Core; abstract class SessionBase @@ -63,8 +63,8 @@ public void TraceProcessingMessage(MessageData data, bool isExtendedSession) Utils.GetTaskEventId(taskMessage.Event), taskMessage.OrchestrationInstance.InstanceId, taskMessage.OrchestrationInstance.ExecutionId, - queueMessage.Id, - Math.Max(0, (int)DateTimeOffset.UtcNow.Subtract(queueMessage.InsertionTime.Value).TotalMilliseconds), + queueMessage.MessageId, + Math.Max(0, (int)DateTimeOffset.UtcNow.Subtract(queueMessage.InsertedOn.Value).TotalMilliseconds), data.SequenceNumber, data.Episode.GetValueOrDefault(-1), isExtendedSession); diff --git a/src/DurableTask.AzureStorage/Messaging/TaskHubQueue.cs b/src/DurableTask.AzureStorage/Messaging/TaskHubQueue.cs index 9dba81579..3188dcec3 100644 --- a/src/DurableTask.AzureStorage/Messaging/TaskHubQueue.cs +++ b/src/DurableTask.AzureStorage/Messaging/TaskHubQueue.cs @@ -19,6 +19,7 @@ namespace DurableTask.AzureStorage.Messaging using System.Text; using System.Threading; using System.Threading.Tasks; + using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Storage; using DurableTask.Core; using DurableTask.Core.History; @@ -44,7 +45,6 @@ abstract class TaskHubQueue this.storageAccountName = azureStorageClient.QueueAccountName; this.settings = azureStorageClient.Settings; - this.storageQueue = this.azureStorageClient.GetQueueReference(queueName); TimeSpan minPollingDelay = TimeSpan.FromMilliseconds(50); @@ -109,8 +109,6 @@ async Task AddMessageAsync(TaskMessage taskMessage, OrchestrationIn string rawContent = await this.messageManager.SerializeMessageDataAsync(data); - QueueMessage queueMessage = new QueueMessage(rawContent); - this.settings.Logger.SendingMessage( outboundTraceActivityId, this.storageAccountName, @@ -127,7 +125,7 @@ async Task AddMessageAsync(TaskMessage taskMessage, OrchestrationIn data.Episode.GetValueOrDefault(-1)); await this.storageQueue.AddMessageAsync( - queueMessage, + rawContent, GetVisibilityDelay(taskMessage), session?.TraceActivityId); @@ -254,7 +252,7 @@ public virtual Task AbandonMessageAsync(MessageData message, SessionBase? sessio this.settings.TaskHubName, eventType, taskEventId, - queueMessage.Id, + queueMessage.MessageId, instanceId, executionId, this.storageQueue.Name, @@ -266,7 +264,7 @@ public virtual Task AbandonMessageAsync(MessageData message, SessionBase? sessio this.settings.TaskHubName, eventType, taskEventId, - queueMessage.Id, + queueMessage.MessageId, instanceId, executionId, this.storageQueue.Name, @@ -288,7 +286,7 @@ public virtual Task AbandonMessageAsync(MessageData message, SessionBase? sessio // Message may have been processed and deleted already. this.HandleMessagingExceptions( e, - queueMessage.Id, + queueMessage.MessageId, instanceId, executionId, eventType, @@ -312,7 +310,7 @@ public async Task RenewMessageAsync(MessageData message, SessionBase session) this.storageQueue.Name, message.TaskMessage.Event.EventType.ToString(), Utils.GetTaskEventId(message.TaskMessage.Event), - queueMessage.Id, + queueMessage.MessageId, queueMessage.PopReceipt, (int)this.MessageVisibilityTimeout.TotalSeconds); @@ -340,7 +338,7 @@ public virtual async Task DeleteMessageAsync(MessageData message, SessionBase? s this.settings.TaskHubName, taskMessage.Event.EventType.ToString(), Utils.GetTaskEventId(taskMessage.Event), - queueMessage.Id, + queueMessage.MessageId, taskMessage.OrchestrationInstance.InstanceId, taskMessage.OrchestrationInstance.ExecutionId, this.storageQueue.Name, @@ -377,7 +375,7 @@ private bool IsMessageGoneException(Exception e) void HandleMessagingExceptions(Exception e, MessageData message, string details) { - string messageId = message.OriginalQueueMessage.Id; + string messageId = message.OriginalQueueMessage.MessageId; string instanceId = message.TaskMessage.OrchestrationInstance.InstanceId; string executionId = message.TaskMessage.OrchestrationInstance.ExecutionId; string eventType = message.TaskMessage.Event.EventType.ToString() ?? string.Empty; diff --git a/src/DurableTask.AzureStorage/Messaging/WorkItemQueue.cs b/src/DurableTask.AzureStorage/Messaging/WorkItemQueue.cs index a420c87bf..512f761f5 100644 --- a/src/DurableTask.AzureStorage/Messaging/WorkItemQueue.cs +++ b/src/DurableTask.AzureStorage/Messaging/WorkItemQueue.cs @@ -16,6 +16,7 @@ namespace DurableTask.AzureStorage.Messaging using System; using System.Threading; using System.Threading.Tasks; + using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Storage; class WorkItemQueue : TaskHubQueue diff --git a/src/DurableTask.AzureStorage/Monitoring/DisconnectedPerformanceMonitor.cs b/src/DurableTask.AzureStorage/Monitoring/DisconnectedPerformanceMonitor.cs index 553ca3e27..9b26db831 100644 --- a/src/DurableTask.AzureStorage/Monitoring/DisconnectedPerformanceMonitor.cs +++ b/src/DurableTask.AzureStorage/Monitoring/DisconnectedPerformanceMonitor.cs @@ -16,10 +16,12 @@ namespace DurableTask.AzureStorage.Monitoring using System; using System.Collections.Generic; using System.Linq; + using System.Net; using System.Text; using System.Threading.Tasks; + using Azure; + using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Storage; - using Microsoft.WindowsAzure.Storage; /// /// Utility class for collecting performance information for a Durable Task hub without actually running inside a Durable Task worker. @@ -49,37 +51,21 @@ public class DisconnectedPerformanceMonitor /// /// The connection string for the Azure Storage account to monitor. /// The name of the task hub within the specified storage account. - public DisconnectedPerformanceMonitor(string storageConnectionString, string taskHub) - : this(CloudStorageAccount.Parse(storageConnectionString), taskHub) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The Azure Storage account to monitor. - /// The name of the task hub within the specified storage account. /// The maximum interval in milliseconds for polling control and work-item queues. - public DisconnectedPerformanceMonitor( - CloudStorageAccount storageAccount, - string taskHub, - int? maxPollingIntervalMilliseconds = null) - : this(storageAccount, GetSettings(taskHub, maxPollingIntervalMilliseconds)) + public DisconnectedPerformanceMonitor(string storageConnectionString, string taskHub, int? maxPollingIntervalMilliseconds = null) + : this(GetSettings(storageConnectionString, taskHub, maxPollingIntervalMilliseconds)) { } /// /// Initializes a new instance of the class. /// - /// The Azure Storage account to monitor. /// The orchestration service settings. - public DisconnectedPerformanceMonitor( - CloudStorageAccount storageAccount, - AzureStorageOrchestrationServiceSettings settings) + public DisconnectedPerformanceMonitor(AzureStorageOrchestrationServiceSettings settings) { this.settings = settings; - this.azureStorageClient = new AzureStorageClient(storageAccount, settings); + this.azureStorageClient = new AzureStorageClient(settings); this.maxPollingLatency = (int)settings.MaxQueuePollingInterval.TotalMilliseconds; this.highLatencyThreshold = Math.Min(this.maxPollingLatency, 1000); @@ -98,10 +84,16 @@ public DisconnectedPerformanceMonitor(string storageConnectionString, string tas internal QueueMetricHistory WorkItemQueueLatencies => this.workItemQueueLatencies; static AzureStorageOrchestrationServiceSettings GetSettings( + string connectionString, string taskHub, int? maxPollingIntervalMilliseconds = null) { - var settings = new AzureStorageOrchestrationServiceSettings { TaskHubName = taskHub }; + var settings = new AzureStorageOrchestrationServiceSettings + { + StorageAccountClientProvider = new StorageAccountClientProvider(connectionString), + TaskHubName = taskHub, + }; + if (maxPollingIntervalMilliseconds != null) { settings.MaxQueuePollingInterval = TimeSpan.FromMilliseconds(maxPollingIntervalMilliseconds.Value); @@ -171,13 +163,13 @@ internal virtual async Task UpdateQueueMetrics() { await Task.WhenAll(tasks); } - catch (StorageException e) when (e.RequestInformation?.HttpStatusCode == 404) + catch (DurableTaskStorageException dtse) when (dtse.InnerException is RequestFailedException rfe && rfe.Status == (int)HttpStatusCode.NotFound) { // The queues are not yet provisioned. this.settings.Logger.GeneralWarning( this.azureStorageClient.QueueAccountName, this.settings.TaskHubName, - $"Task hub has not been provisioned: {e.RequestInformation.ExtendedErrorInformation?.ErrorMessage}"); + $"Task hub has not been provisioned: {rfe.Message}"); return false; } @@ -231,21 +223,20 @@ async Task GetQueueMetricsAsync(Queue queue) static async Task GetQueueLatencyAsync(Queue queue) { DateTimeOffset now = DateTimeOffset.UtcNow; - QueueMessage firstMessage = await queue.PeekMessageAsync(); + PeekedMessage firstMessage = await queue.PeekMessageAsync(); if (firstMessage == null) { return TimeSpan.MinValue; } // Make sure we always return a non-negative timespan in the success case. - TimeSpan latency = now.Subtract(firstMessage.InsertionTime.GetValueOrDefault(now)); + TimeSpan latency = now.Subtract(firstMessage.InsertedOn.GetValueOrDefault(now)); return latency < TimeSpan.Zero ? TimeSpan.Zero : latency; } - static async Task GetQueueLengthAsync(Queue queue) + static Task GetQueueLengthAsync(Queue queue) { - await queue.FetchAttributesAsync(); - return queue.ApproximateMessageCount.GetValueOrDefault(0); + return queue.GetApproximateMessagesCountAsync(); } struct QueueMetric @@ -264,12 +255,11 @@ protected virtual async Task GetWorkItemQueueStatusAsync() DateTimeOffset now = DateTimeOffset.Now; - Task fetchTask = workItemQueue.FetchAttributesAsync(); - Task peekTask = workItemQueue.PeekMessageAsync(); - await Task.WhenAll(fetchTask, peekTask); + Task peekTask = workItemQueue.PeekMessageAsync(); + Task lengthTask = workItemQueue.GetApproximateMessagesCountAsync(); + await Task.WhenAll(lengthTask, peekTask); - int queueLength = workItemQueue.ApproximateMessageCount.GetValueOrDefault(0); - TimeSpan age = now.Subtract((peekTask.Result?.InsertionTime).GetValueOrDefault(now)); + TimeSpan age = now.Subtract((peekTask.Result?.InsertedOn).GetValueOrDefault(now)); if (age < TimeSpan.Zero) { age = TimeSpan.Zero; @@ -277,7 +267,7 @@ protected virtual async Task GetWorkItemQueueStatusAsync() return new WorkItemQueueData { - QueueLength = queueLength, + QueueLength = lengthTask.Result, FirstMessageAge = age, }; } @@ -299,9 +289,7 @@ protected virtual async Task GetAggregateControlQueueLengthAsy // We treat all control queues like one big queue and sum the lengths together. foreach (Queue queue in controlQueues) { - await queue.FetchAttributesAsync(); - int queueLength = queue.ApproximateMessageCount.GetValueOrDefault(0); - result.AggregateQueueLength += queueLength; + result.AggregateQueueLength += await queue.GetApproximateMessagesCountAsync(); } return result; @@ -597,22 +585,6 @@ public override string ToString() builder.Remove(builder.Length - 1, 1).Append(']'); return builder.ToString(); } - - static void ThrowIfNegative(string paramName, double value) - { - if (value < 0.0) - { - throw new ArgumentOutOfRangeException(paramName, value, $"{paramName} cannot be negative."); - } - } - - static void ThrowIfPositive(string paramName, double value) - { - if (value > 0.0) - { - throw new ArgumentOutOfRangeException(paramName, value, $"{paramName} cannot be positive."); - } - } } } } diff --git a/src/DurableTask.AzureStorage/OrchestrationInstanceStatus.cs b/src/DurableTask.AzureStorage/OrchestrationInstanceStatus.cs index b12d11712..5511c39ff 100644 --- a/src/DurableTask.AzureStorage/OrchestrationInstanceStatus.cs +++ b/src/DurableTask.AzureStorage/OrchestrationInstanceStatus.cs @@ -14,12 +14,13 @@ namespace DurableTask.AzureStorage { using System; - using Microsoft.WindowsAzure.Storage.Table; + using Azure; + using Azure.Data.Tables; /// /// Table Entity Representation of an Orchestration Instance's Status /// - class OrchestrationInstanceStatus : TableEntity + class OrchestrationInstanceStatus : ITableEntity { public string ExecutionId { get; set; } public string Name { get; set; } @@ -33,5 +34,9 @@ class OrchestrationInstanceStatus : TableEntity public string RuntimeStatus { get; set; } public DateTime? ScheduledStartTime { get; set; } public int Generation { get; set; } + public string PartitionKey { get; set; } + public string RowKey { get; set; } + public DateTimeOffset? Timestamp { get; set; } + public ETag ETag { get; set; } } } diff --git a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs index e18853031..c2f6c726b 100644 --- a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs +++ b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs @@ -19,10 +19,11 @@ namespace DurableTask.AzureStorage using System.Linq; using System.Threading; using System.Threading.Tasks; + using Azure; + using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Messaging; using DurableTask.AzureStorage.Monitoring; using DurableTask.AzureStorage.Partitioning; - using DurableTask.AzureStorage.Storage; using DurableTask.AzureStorage.Tracking; using DurableTask.Core; using DurableTask.Core.History; @@ -228,9 +229,9 @@ async Task DequeueLoop(string partitionId, ControlQueue controlQueue, Cancellati // Terminology: // "Local" -> the instance ID info comes from the local copy of the message we're examining // "Remote" -> the instance ID info comes from the Instances table that we're querying - IList instances = await this.trackingStore.GetStateAsync(instanceIds); - IDictionary remoteOrchestrationsById = - instances.ToDictionary(o => o.OrchestrationInstance.InstanceId); + IAsyncEnumerable instances = this.trackingStore.GetStateAsync(instanceIds, cancellationToken); + IDictionary remoteOrchestrationsById = + await instances.ToDictionaryAsync(o => o.OrchestrationInstance.InstanceId, cancellationToken); foreach (MessageData message in executionStartedMessages) { @@ -290,7 +291,7 @@ async Task DequeueLoop(string partitionId, ControlQueue controlQueue, Cancellati this.settings.TaskHubName, msg.TaskMessage.Event.EventType.ToString(), Utils.GetTaskEventId(msg.TaskMessage.Event), - msg.OriginalQueueMessage.Id, + msg.OriginalQueueMessage.MessageId, msg.TaskMessage.OrchestrationInstance.InstanceId, msg.TaskMessage.OrchestrationInstance.ExecutionId, controlQueue.Name, @@ -329,7 +330,7 @@ bool IsScheduledAfterInstanceUpdate(MessageData msg, OrchestrationState? remoteI } QueueMessage queueMessage = msg.OriginalQueueMessage; - if (queueMessage.DequeueCount <= 1 || !queueMessage.NextVisibleTime.HasValue) + if (queueMessage.DequeueCount <= 1 || !queueMessage.NextVisibleOn.HasValue) { // We can't use the initial insert time and instead must rely on a re-insertion time, // which is only available to use after the first dequeue count. @@ -338,7 +339,7 @@ bool IsScheduledAfterInstanceUpdate(MessageData msg, OrchestrationState? remoteI // This calculation assumes that the value of ControlQueueVisibilityTimeout did not change // in any meaningful way between the time the message was inserted and now. - DateTime latestReinsertionTime = queueMessage.NextVisibleTime.Value.Subtract(this.settings.ControlQueueVisibilityTimeout).DateTime; + DateTime latestReinsertionTime = queueMessage.NextVisibleOn.Value.Subtract(this.settings.ControlQueueVisibilityTimeout).DateTime; return latestReinsertionTime > remoteInstance.CreatedTime; } @@ -681,7 +682,7 @@ public PendingMessageBatch(ControlQueue controlQueue, string instanceId, string? } } - public string? ETag { get; set; } + public ETag? ETag { get; set; } public DateTime LastCheckpointTime { get; set; } } } diff --git a/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs b/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs index 5ea594da4..35550e1e3 100644 --- a/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs +++ b/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs @@ -554,7 +554,6 @@ async Task GetAppLeaseInfoAsync() { if (await this.appLeaseInfoBlob.ExistsAsync()) { - await appLeaseInfoBlob.FetchAttributesAsync(); string serializedEventHubInfo = await this.appLeaseInfoBlob.DownloadTextAsync(); return Utils.DeserializeFromJson(serializedEventHubInfo); } diff --git a/src/DurableTask.AzureStorage/Partitioning/BlobLease.cs b/src/DurableTask.AzureStorage/Partitioning/BlobPartitionLease.cs similarity index 74% rename from src/DurableTask.AzureStorage/Partitioning/BlobLease.cs rename to src/DurableTask.AzureStorage/Partitioning/BlobPartitionLease.cs index 99b08ffb0..26a01a8a2 100644 --- a/src/DurableTask.AzureStorage/Partitioning/BlobLease.cs +++ b/src/DurableTask.AzureStorage/Partitioning/BlobPartitionLease.cs @@ -13,24 +13,25 @@ namespace DurableTask.AzureStorage.Partitioning { + using System.Threading; using System.Threading.Tasks; using DurableTask.AzureStorage.Storage; using Newtonsoft.Json; - class BlobLease : Lease + class BlobPartitionLease : Lease { - public BlobLease() + public BlobPartitionLease() : base() { } - public BlobLease(Blob leaseBlob) + public BlobPartitionLease(Blob leaseBlob) : this() { this.Blob = leaseBlob; } - public BlobLease(BlobLease source) + public BlobPartitionLease(BlobPartitionLease source) : base(source) { this.Blob = source.Blob; @@ -39,7 +40,9 @@ public BlobLease(BlobLease source) [JsonIgnore] public Blob Blob { get; set; } - [JsonIgnore] - public override bool IsExpired => !Blob.IsLeased; + public override async Task IsExpiredAsync(CancellationToken cancellationToken = default) + { + return !await this.Blob.IsLeasedAsync(cancellationToken); + } } } diff --git a/src/DurableTask.AzureStorage/Partitioning/BlobLeaseManager.cs b/src/DurableTask.AzureStorage/Partitioning/BlobPartitionLeaseManager.cs similarity index 72% rename from src/DurableTask.AzureStorage/Partitioning/BlobLeaseManager.cs rename to src/DurableTask.AzureStorage/Partitioning/BlobPartitionLeaseManager.cs index c76a741c8..07335ab06 100644 --- a/src/DurableTask.AzureStorage/Partitioning/BlobLeaseManager.cs +++ b/src/DurableTask.AzureStorage/Partitioning/BlobPartitionLeaseManager.cs @@ -19,12 +19,14 @@ namespace DurableTask.AzureStorage.Partitioning using System.IO; using System.Linq; using System.Net; + using System.Runtime.CompilerServices; using System.Text; + using System.Threading; using System.Threading.Tasks; + using Azure; using DurableTask.AzureStorage.Storage; - using Newtonsoft.Json; - sealed class BlobLeaseManager : ILeaseManager + sealed class BlobPartitionLeaseManager : ILeaseManager { const string TaskHubInfoBlobName = "taskhub.json"; @@ -40,7 +42,7 @@ sealed class BlobLeaseManager : ILeaseManager string blobDirectoryName; Blob taskHubInfoBlob; - public BlobLeaseManager( + public BlobPartitionLeaseManager( AzureStorageClient azureStorageClient, string leaseContainerName, string leaseType) @@ -57,44 +59,38 @@ sealed class BlobLeaseManager : ILeaseManager this.Initialize(); } - public Task LeaseStoreExistsAsync() + public Task LeaseStoreExistsAsync(CancellationToken cancellationToken = default) { - return taskHubContainer.ExistsAsync(); + return taskHubContainer.ExistsAsync(cancellationToken); } - public async Task CreateLeaseStoreIfNotExistsAsync(TaskHubInfo eventHubInfo, bool checkIfStale) + public async Task CreateLeaseStoreIfNotExistsAsync(TaskHubInfo eventHubInfo, bool checkIfStale, CancellationToken cancellationToken = default) { - bool result = false; - result = await taskHubContainer.CreateIfNotExistsAsync(); + bool result = await taskHubContainer.CreateIfNotExistsAsync(cancellationToken); - await this.GetOrCreateTaskHubInfoAsync(eventHubInfo, checkIfStale: checkIfStale); + await this.GetOrCreateTaskHubInfoAsync(eventHubInfo, checkIfStale: checkIfStale, cancellationToken: cancellationToken); return result; } - public async Task> ListLeasesAsync() + public async IAsyncEnumerable ListLeasesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) { - var blobLeases = new List(); - - IEnumerable blobs = await this.taskHubContainer.ListBlobsAsync(this.blobDirectoryName); - - var downloadTasks = new List>(); - foreach (Blob blob in blobs) + await foreach (Page page in this.taskHubContainer.ListBlobsAsync(this.blobDirectoryName, cancellationToken: cancellationToken).AsPages()) { - downloadTasks.Add(this.DownloadLeaseBlob(blob)); - } - - await Task.WhenAll(downloadTasks); + // Start each of the Tasks in parallel + Task[] downloadTasks = page.Values.Select(b => this.DownloadLeaseBlob(b, cancellationToken)).ToArray(); - blobLeases.AddRange(downloadTasks.Select(t => t.Result)); - - return blobLeases; + foreach (Task downloadTask in downloadTasks) + { + yield return await downloadTask; + } + } } - public async Task CreateLeaseIfNotExistAsync(string partitionId) + public async Task CreateLeaseIfNotExistAsync(string partitionId, CancellationToken cancellationToken = default) { Blob leaseBlob = this.taskHubContainer.GetBlobReference(partitionId, this.blobDirectoryName); - BlobLease lease = new BlobLease(leaseBlob) { PartitionId = partitionId }; + BlobPartitionLease lease = new BlobPartitionLease(leaseBlob) { PartitionId = partitionId }; string serializedLease = Utils.SerializeToJson(lease); try { @@ -110,7 +106,7 @@ public async Task CreateLeaseIfNotExistAsync(string partitionId) this.blobDirectoryName, partitionId)); - await leaseBlob.UploadTextAsync(serializedLease, ifDoesntExist: true); + await leaseBlob.UploadTextAsync(serializedLease, ifDoesntExist: true, cancellationToken: cancellationToken); } catch (DurableTaskStorageException se) { @@ -134,23 +130,23 @@ public async Task CreateLeaseIfNotExistAsync(string partitionId) } } - public async Task GetLeaseAsync(string partitionId) + public async Task GetLeaseAsync(string partitionId, CancellationToken cancellationToken = default) { Blob leaseBlob = this.taskHubContainer.GetBlobReference(partitionId, this.blobDirectoryName); - if (await leaseBlob.ExistsAsync()) + if (await leaseBlob.ExistsAsync(cancellationToken)) { - return await this.DownloadLeaseBlob(leaseBlob); + return await this.DownloadLeaseBlob(leaseBlob, cancellationToken); } return null; } - public async Task RenewAsync(BlobLease lease) + public async Task RenewAsync(BlobPartitionLease lease, CancellationToken cancellationToken = default) { Blob leaseBlob = lease.Blob; try { - await leaseBlob.RenewLeaseAsync(lease.Token); + await leaseBlob.RenewLeaseAsync(lease.Token, cancellationToken); } catch (DurableTaskStorageException storageException) { @@ -160,25 +156,25 @@ public async Task RenewAsync(BlobLease lease) return true; } - public async Task AcquireAsync(BlobLease lease, string owner) + public async Task AcquireAsync(BlobPartitionLease lease, string owner, CancellationToken cancellationToken = default) { Blob leaseBlob = lease.Blob; try { string newLeaseId = Guid.NewGuid().ToString(); - if (leaseBlob.IsLeased) + if (await leaseBlob.IsLeasedAsync(cancellationToken)) { - lease.Token = await leaseBlob.ChangeLeaseAsync(newLeaseId, currentLeaseId: lease.Token); + lease.Token = await leaseBlob.ChangeLeaseAsync(newLeaseId, currentLeaseId: lease.Token, cancellationToken: cancellationToken); } else { - lease.Token = await leaseBlob.AcquireLeaseAsync(this.leaseInterval, newLeaseId); + lease.Token = await leaseBlob.AcquireLeaseAsync(this.leaseInterval, newLeaseId, cancellationToken: cancellationToken); } lease.Owner = owner; // Increment Epoch each time lease is acquired or stolen by new host lease.Epoch += 1; - await leaseBlob.UploadTextAsync(Utils.SerializeToJson(lease), leaseId: lease.Token); + await leaseBlob.UploadTextAsync(Utils.SerializeToJson(lease), leaseId: lease.Token, cancellationToken: cancellationToken); } catch (DurableTaskStorageException storageException) { @@ -188,18 +184,18 @@ public async Task AcquireAsync(BlobLease lease, string owner) return true; } - public async Task ReleaseAsync(BlobLease lease) + public async Task ReleaseAsync(BlobPartitionLease lease, CancellationToken cancellationToken = default) { Blob leaseBlob = lease.Blob; try { string leaseId = lease.Token; - BlobLease copy = new BlobLease(lease); + BlobPartitionLease copy = new BlobPartitionLease(lease); copy.Token = null; copy.Owner = null; - await leaseBlob.UploadTextAsync(Utils.SerializeToJson(copy), leaseId); - await leaseBlob.ReleaseLeaseAsync(leaseId); + await leaseBlob.UploadTextAsync(Utils.SerializeToJson(copy), leaseId, cancellationToken: cancellationToken); + await leaseBlob.ReleaseLeaseAsync(leaseId, cancellationToken); } catch (DurableTaskStorageException storageException) { @@ -209,17 +205,17 @@ public async Task ReleaseAsync(BlobLease lease) return true; } - public async Task DeleteAsync(BlobLease lease) + public Task DeleteAsync(BlobPartitionLease lease, CancellationToken cancellationToken = default) { - await lease.Blob.DeleteIfExistsAsync(); + return lease.Blob.DeleteIfExistsAsync(cancellationToken); } - public async Task DeleteAllAsync() + public Task DeleteAllAsync(CancellationToken cancellationToken = default) { - await this.taskHubContainer.DeleteIfExistsAsync(); + return this.taskHubContainer.DeleteIfExistsAsync(cancellationToken: cancellationToken); } - public async Task UpdateAsync(BlobLease lease) + public async Task UpdateAsync(BlobPartitionLease lease, CancellationToken cancellationToken = default) { if (lease == null || string.IsNullOrWhiteSpace(lease.Token)) { @@ -230,7 +226,7 @@ public async Task UpdateAsync(BlobLease lease) try { // First renew the lease to make sure checkpoint will go through - await leaseBlob.RenewLeaseAsync(lease.Token); + await leaseBlob.RenewLeaseAsync(lease.Token, cancellationToken); } catch (DurableTaskStorageException storageException) { @@ -239,7 +235,7 @@ public async Task UpdateAsync(BlobLease lease) try { - await leaseBlob.UploadTextAsync(Utils.SerializeToJson(lease), lease.Token); + await leaseBlob.UploadTextAsync(Utils.SerializeToJson(lease), lease.Token, cancellationToken: cancellationToken); } catch (DurableTaskStorageException storageException) { @@ -249,12 +245,12 @@ public async Task UpdateAsync(BlobLease lease) return true; } - public async Task CreateTaskHubInfoIfNotExistAsync(TaskHubInfo taskHubInfo) + public async Task CreateTaskHubInfoIfNotExistAsync(TaskHubInfo taskHubInfo, CancellationToken cancellationToken = default) { string serializedInfo = Utils.SerializeToJson(taskHubInfo); try { - await this.taskHubInfoBlob.UploadTextAsync(serializedInfo, ifDoesntExist: true); + await this.taskHubInfoBlob.UploadTextAsync(serializedInfo, ifDoesntExist: true, cancellationToken: cancellationToken); } catch (DurableTaskStorageException) { @@ -263,9 +259,9 @@ public async Task CreateTaskHubInfoIfNotExistAsync(TaskHubInfo taskHubInfo) } } - internal async Task GetOrCreateTaskHubInfoAsync(TaskHubInfo newTaskHubInfo, bool checkIfStale) + internal async Task GetOrCreateTaskHubInfoAsync(TaskHubInfo newTaskHubInfo, bool checkIfStale, CancellationToken cancellationToken = default) { - TaskHubInfo currentTaskHubInfo = await this.GetTaskHubInfoAsync(); + TaskHubInfo currentTaskHubInfo = await this.GetTaskHubInfoAsync(cancellationToken); if (currentTaskHubInfo != null) { if (checkIfStale && IsStale(currentTaskHubInfo, newTaskHubInfo)) @@ -280,7 +276,7 @@ internal async Task GetOrCreateTaskHubInfoAsync(TaskHubInfo newTask try { string serializedInfo = Utils.SerializeToJson(newTaskHubInfo); - await this.taskHubInfoBlob.UploadTextAsync(serializedInfo); + await this.taskHubInfoBlob.UploadTextAsync(serializedInfo, cancellationToken: cancellationToken); } catch (DurableTaskStorageException) { @@ -292,7 +288,7 @@ internal async Task GetOrCreateTaskHubInfoAsync(TaskHubInfo newTask return currentTaskHubInfo; } - await this.CreateTaskHubInfoIfNotExistAsync(newTaskHubInfo); + await this.CreateTaskHubInfoIfNotExistAsync(newTaskHubInfo, cancellationToken); return newTaskHubInfo; } @@ -309,19 +305,18 @@ void Initialize() this.taskHubInfoBlob = this.taskHubContainer.GetBlobReference(TaskHubInfoBlobName); } - async Task GetTaskHubInfoAsync() + async Task GetTaskHubInfoAsync(CancellationToken cancellationToken) { - if (await this.taskHubInfoBlob.ExistsAsync()) + if (await this.taskHubInfoBlob.ExistsAsync(cancellationToken)) { - await taskHubInfoBlob.FetchAttributesAsync(); - string serializedEventHubInfo = await this.taskHubInfoBlob.DownloadTextAsync(); + string serializedEventHubInfo = await this.taskHubInfoBlob.DownloadTextAsync(cancellationToken); return Utils.DeserializeFromJson(serializedEventHubInfo); } return null; } - async Task DownloadLeaseBlob(Blob blob) + async Task DownloadLeaseBlob(Blob blob, CancellationToken cancellationToken) { string serializedLease = null; var buffer = SimpleBufferManager.Shared.TakeBuffer(SimpleBufferManager.SmallBufferSize); @@ -329,7 +324,7 @@ async Task DownloadLeaseBlob(Blob blob) { using (var memoryStream = new MemoryStream(buffer)) { - await blob.DownloadToStreamAsync(memoryStream); + await blob.DownloadToStreamAsync(memoryStream, cancellationToken); serializedLease = Encoding.UTF8.GetString(buffer, 0, (int)memoryStream.Position); } } @@ -338,11 +333,9 @@ async Task DownloadLeaseBlob(Blob blob) SimpleBufferManager.Shared.ReturnBuffer(buffer); } - BlobLease deserializedLease = Utils.DeserializeFromJson(serializedLease); + BlobPartitionLease deserializedLease = Utils.DeserializeFromJson(serializedLease); deserializedLease.Blob = blob; - // Workaround: for some reason storage client reports incorrect blob properties after downloading the blob - await blob.FetchAttributesAsync(); return deserializedLease; } diff --git a/src/DurableTask.AzureStorage/Partitioning/ILeaseManager.cs b/src/DurableTask.AzureStorage/Partitioning/ILeaseManager.cs index 1ad5b0c82..cf24b7291 100644 --- a/src/DurableTask.AzureStorage/Partitioning/ILeaseManager.cs +++ b/src/DurableTask.AzureStorage/Partitioning/ILeaseManager.cs @@ -14,30 +14,31 @@ namespace DurableTask.AzureStorage.Partitioning { using System.Collections.Generic; + using System.Threading; using System.Threading.Tasks; interface ILeaseManager where T : Lease { - Task LeaseStoreExistsAsync(); + Task LeaseStoreExistsAsync(CancellationToken cancellationToken = default); - Task CreateLeaseStoreIfNotExistsAsync(TaskHubInfo eventHubInfo, bool checkIfStale = true); + Task CreateLeaseStoreIfNotExistsAsync(TaskHubInfo eventHubInfo, bool checkIfStale = true, CancellationToken cancellationToken = default); - Task> ListLeasesAsync(); + IAsyncEnumerable ListLeasesAsync(CancellationToken cancellationToken = default); - Task CreateLeaseIfNotExistAsync(string partitionId); + Task CreateLeaseIfNotExistAsync(string partitionId, CancellationToken cancellationToken = default); - Task GetLeaseAsync(string partitionId); + Task GetLeaseAsync(string partitionId, CancellationToken cancellationToken = default); - Task RenewAsync(T lease); + Task RenewAsync(T lease, CancellationToken cancellationToken = default); - Task AcquireAsync(T lease, string owner); + Task AcquireAsync(T lease, string owner, CancellationToken cancellationToken = default); - Task ReleaseAsync(T lease); + Task ReleaseAsync(T lease, CancellationToken cancellationToken = default); - Task DeleteAsync(T lease); + Task DeleteAsync(T lease, CancellationToken cancellationToken = default); - Task DeleteAllAsync(); + Task DeleteAllAsync(CancellationToken cancellationToken = default); - Task UpdateAsync(T lease); + Task UpdateAsync(T lease, CancellationToken cancellationToken = default); } } diff --git a/src/DurableTask.AzureStorage/Partitioning/IPartitionManager.cs b/src/DurableTask.AzureStorage/Partitioning/IPartitionManager.cs index 3175cad10..feb3a99bd 100644 --- a/src/DurableTask.AzureStorage/Partitioning/IPartitionManager.cs +++ b/src/DurableTask.AzureStorage/Partitioning/IPartitionManager.cs @@ -14,6 +14,7 @@ namespace DurableTask.AzureStorage.Partitioning { using System.Collections.Generic; + using System.Threading; using System.Threading.Tasks; interface IPartitionManager @@ -28,6 +29,6 @@ interface IPartitionManager Task DeleteLeases(); - Task> GetOwnershipBlobLeases(); + IAsyncEnumerable GetOwnershipBlobLeases(CancellationToken cancellationToken = default); } } diff --git a/src/DurableTask.AzureStorage/Partitioning/Lease.cs b/src/DurableTask.AzureStorage/Partitioning/Lease.cs index 9d0d248b3..78bdfee98 100644 --- a/src/DurableTask.AzureStorage/Partitioning/Lease.cs +++ b/src/DurableTask.AzureStorage/Partitioning/Lease.cs @@ -11,6 +11,9 @@ // limitations under the License. // ---------------------------------------------------------------------------------- +using System.Threading; +using System.Threading.Tasks; + namespace DurableTask.AzureStorage.Partitioning { /// Contains partition ownership information. @@ -53,7 +56,10 @@ public Lease(Lease source) /// Determines whether the lease is expired. /// true if the lease is expired; otherwise, false. - public virtual bool IsExpired => false; + public virtual Task IsExpiredAsync(CancellationToken cancellationToken = default) + { + return Task.FromResult(false); + } /// Determines whether this instance is equal to the specified object. /// The object to compare. diff --git a/src/DurableTask.AzureStorage/Partitioning/LeaseCollectionBalancer.cs b/src/DurableTask.AzureStorage/Partitioning/LeaseCollectionBalancer.cs index 063a90178..a2b89a42e 100644 --- a/src/DurableTask.AzureStorage/Partitioning/LeaseCollectionBalancer.cs +++ b/src/DurableTask.AzureStorage/Partitioning/LeaseCollectionBalancer.cs @@ -51,7 +51,6 @@ sealed class LeaseCollectionBalancer where T : Lease LeaseCollectionBalancerOptions options, Func shouldAquireLeaseDelegate = null, Func shouldRenewLeaseDelegate = null) - { this.leaseType = leaseType; this.accountName = blobAccountName; @@ -82,7 +81,7 @@ private static bool DefaultLeaseDecisionDelegate(string leaseId) public async Task InitializeAsync() { var leases = new List(); - foreach (T lease in await this.leaseManager.ListLeasesAsync()) + await foreach (T lease in this.leaseManager.ListLeasesAsync()) { if (string.Compare(lease.Owner, this.workerName, StringComparison.OrdinalIgnoreCase) == 0) { @@ -331,8 +330,7 @@ async Task LeaseTakerAsync() var workerToShardCount = new Dictionary(); var expiredLeases = new List(); - var allLeases = await this.leaseManager.ListLeasesAsync(); - foreach (T lease in allLeases) + await foreach (T lease in this.leaseManager.ListLeasesAsync()) { if (!this.shouldAquireLeaseDelegate(lease.PartitionId)) { @@ -346,7 +344,7 @@ async Task LeaseTakerAsync() } allShards.Add(lease.PartitionId, lease); - if (lease.IsExpired || string.IsNullOrWhiteSpace(lease.Owner)) + if (await lease.IsExpiredAsync() || string.IsNullOrWhiteSpace(lease.Owner)) { expiredLeases.Add(lease); } diff --git a/src/DurableTask.AzureStorage/Partitioning/LegacyPartitionManager.cs b/src/DurableTask.AzureStorage/Partitioning/LegacyPartitionManager.cs index eaab23152..6b5bc9b17 100644 --- a/src/DurableTask.AzureStorage/Partitioning/LegacyPartitionManager.cs +++ b/src/DurableTask.AzureStorage/Partitioning/LegacyPartitionManager.cs @@ -16,9 +16,10 @@ namespace DurableTask.AzureStorage.Partitioning using System; using System.Collections.Generic; using System.Runtime.ExceptionServices; + using System.Threading; using System.Threading.Tasks; + using Azure; using DurableTask.AzureStorage.Storage; - using Microsoft.WindowsAzure.Storage; class LegacyPartitionManager : IPartitionManager { @@ -26,8 +27,8 @@ class LegacyPartitionManager : IPartitionManager readonly AzureStorageClient azureStorageClient; readonly AzureStorageOrchestrationServiceSettings settings; - readonly BlobLeaseManager leaseManager; - readonly LeaseCollectionBalancer leaseCollectionManager; + readonly BlobPartitionLeaseManager leaseManager; + readonly LeaseCollectionBalancer leaseCollectionManager; public LegacyPartitionManager( AzureStorageOrchestrationService service, @@ -40,7 +41,7 @@ class LegacyPartitionManager : IPartitionManager this.azureStorageClient, "default"); - this.leaseCollectionManager = new LeaseCollectionBalancer( + this.leaseCollectionManager = new LeaseCollectionBalancer( "default", settings, this.azureStorageClient.BlobAccountName, @@ -73,8 +74,8 @@ Task IPartitionManager.DeleteLeases() { foreach (Exception e in t.Exception.InnerExceptions) { - StorageException storageException = e as StorageException; - if (storageException == null || storageException.RequestInformation.HttpStatusCode != 404) + RequestFailedException storageException = e as RequestFailedException; + if (storageException == null || storageException.Status != 404) { ExceptionDispatchInfo.Capture(e).Throw(); } @@ -83,9 +84,9 @@ Task IPartitionManager.DeleteLeases() }); } - Task> IPartitionManager.GetOwnershipBlobLeases() + IAsyncEnumerable IPartitionManager.GetOwnershipBlobLeases(CancellationToken cancellationToken) { - return this.leaseManager.ListLeasesAsync(); + return this.leaseManager.ListLeasesAsync(cancellationToken); } async Task IPartitionManager.StartAsync() diff --git a/src/DurableTask.AzureStorage/Partitioning/SafePartitionManager.cs b/src/DurableTask.AzureStorage/Partitioning/SafePartitionManager.cs index c0122c244..2dd16ffa8 100644 --- a/src/DurableTask.AzureStorage/Partitioning/SafePartitionManager.cs +++ b/src/DurableTask.AzureStorage/Partitioning/SafePartitionManager.cs @@ -16,9 +16,10 @@ namespace DurableTask.AzureStorage.Partitioning using System; using System.Collections.Generic; using System.Runtime.ExceptionServices; + using System.Threading; using System.Threading.Tasks; + using Azure; using DurableTask.AzureStorage.Storage; - using Microsoft.WindowsAzure.Storage; class SafePartitionManager : IPartitionManager { @@ -27,11 +28,11 @@ class SafePartitionManager : IPartitionManager readonly AzureStorageOrchestrationServiceSettings settings; readonly OrchestrationSessionManager sessionManager; - readonly BlobLeaseManager intentLeaseManager; - readonly LeaseCollectionBalancer intentLeaseCollectionManager; + readonly BlobPartitionLeaseManager intentLeaseManager; + readonly LeaseCollectionBalancer intentLeaseCollectionManager; - readonly BlobLeaseManager ownershipLeaseManager; - readonly LeaseCollectionBalancer ownershipLeaseCollectionManager; + readonly BlobPartitionLeaseManager ownershipLeaseManager; + readonly LeaseCollectionBalancer ownershipLeaseCollectionManager; IDisposable intentLeaseSubscription; IDisposable ownershipLeaseSubscription; @@ -50,7 +51,7 @@ class SafePartitionManager : IPartitionManager this.azureStorageClient, "intent"); - this.intentLeaseCollectionManager = new LeaseCollectionBalancer( + this.intentLeaseCollectionManager = new LeaseCollectionBalancer( "intent", settings, this.azureStorageClient.BlobAccountName, @@ -68,7 +69,7 @@ class SafePartitionManager : IPartitionManager this.azureStorageClient, "ownership"); - this.ownershipLeaseCollectionManager = new LeaseCollectionBalancer( + this.ownershipLeaseCollectionManager = new LeaseCollectionBalancer( "ownership", this.settings, this.azureStorageClient.BlobAccountName, @@ -86,9 +87,9 @@ class SafePartitionManager : IPartitionManager || this.sessionManager.IsControlQueueProcessingMessages(leaseKey)); } - Task> IPartitionManager.GetOwnershipBlobLeases() + IAsyncEnumerable IPartitionManager.GetOwnershipBlobLeases(CancellationToken cancellationToken) { - return this.ownershipLeaseManager.ListLeasesAsync(); + return this.ownershipLeaseManager.ListLeasesAsync(cancellationToken); } Task IPartitionManager.CreateLeaseStore() @@ -112,8 +113,8 @@ Task IPartitionManager.DeleteLeases() { foreach (Exception e in t.Exception.InnerExceptions) { - StorageException storageException = e as StorageException; - if (storageException == null || storageException.RequestInformation.HttpStatusCode != 404) + RequestFailedException storageException = e as RequestFailedException; + if (storageException == null || storageException.Status != 404) { ExceptionDispatchInfo.Capture(e).Throw(); } diff --git a/src/DurableTask.AzureStorage/SimpleBufferManager.cs b/src/DurableTask.AzureStorage/SimpleBufferManager.cs index 972a42457..44e64bec7 100644 --- a/src/DurableTask.AzureStorage/SimpleBufferManager.cs +++ b/src/DurableTask.AzureStorage/SimpleBufferManager.cs @@ -16,13 +16,12 @@ namespace DurableTask.AzureStorage using System; using System.Collections.Concurrent; using System.Threading; - using Microsoft.WindowsAzure.Storage; /// /// Simple buffer manager intended for use with Azure Storage SDK and compression code. /// It is not intended to be robust enough for external use. /// - class SimpleBufferManager : IBufferManager + class SimpleBufferManager { internal const int MaxBufferSize = 1024 * 1024; // 1 MB const int DefaultBufferSize = 64 * 1024; // 64 KB diff --git a/src/DurableTask.AzureStorage/Storage/AsyncPageableAsyncProjection.cs b/src/DurableTask.AzureStorage/Storage/AsyncPageableAsyncProjection.cs new file mode 100644 index 000000000..2a1bf7a7c --- /dev/null +++ b/src/DurableTask.AzureStorage/Storage/AsyncPageableAsyncProjection.cs @@ -0,0 +1,62 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Storage +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Azure; + + sealed class AsyncPageableAsyncProjection : AsyncPageable + where TSource : notnull + where TResult : notnull + { + readonly AsyncPageable source; + readonly Func> selector; + + public AsyncPageableAsyncProjection(AsyncPageable source, Func> selector) + { + this.source = source ?? throw new ArgumentNullException(nameof(source)); + this.selector = selector ?? throw new ArgumentNullException(nameof(selector)); + } + + public override IAsyncEnumerable> AsPages(string? continuationToken = null, int? pageSizeHint = null) => + new AsyncPageableProjectionEnumerable(this.source.AsPages(continuationToken, pageSizeHint), this.selector); + + sealed class AsyncPageableProjectionEnumerable : IAsyncEnumerable> + { + readonly IAsyncEnumerable> source; + readonly Func> selector; + + public AsyncPageableProjectionEnumerable(IAsyncEnumerable> source, Func> selector) + { + this.source = source; + this.selector = selector; + } + + public async IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default(CancellationToken)) + { + await foreach (Page page in this.source.WithCancellation(cancellationToken)) + { + yield return Page.FromValues( + await page.Values.ToAsyncEnumerable().SelectAwaitWithCancellation(this.selector).ToListAsync(cancellationToken), + page.ContinuationToken, + page.GetRawResponse()); + } + } + } + } +} diff --git a/src/DurableTask.AzureStorage/Storage/AsyncPageableProjection.cs b/src/DurableTask.AzureStorage/Storage/AsyncPageableProjection.cs new file mode 100644 index 000000000..57d51df61 --- /dev/null +++ b/src/DurableTask.AzureStorage/Storage/AsyncPageableProjection.cs @@ -0,0 +1,59 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Storage +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Azure; + + sealed class AsyncPageableProjection : AsyncPageable + where TSource : notnull + where TResult : notnull + { + readonly AsyncPageable source; + readonly Func selector; + + public AsyncPageableProjection(AsyncPageable source, Func selector) + { + this.source = source ?? throw new ArgumentNullException(nameof(source)); + this.selector = selector ?? throw new ArgumentNullException(nameof(selector)); + } + + public override IAsyncEnumerable> AsPages(string? continuationToken = null, int? pageSizeHint = null) => + new AsyncPageableProjectionEnumerable(this.source.AsPages(continuationToken, pageSizeHint), this.selector); + + sealed class AsyncPageableProjectionEnumerable : IAsyncEnumerable> + { + readonly IAsyncEnumerable> source; + readonly Func selector; + + public AsyncPageableProjectionEnumerable(IAsyncEnumerable> source, Func selector) + { + this.source = source; + this.selector = selector; + } + + public async IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default(CancellationToken)) + { + await foreach (Page page in this.source.WithCancellation(cancellationToken)) + { + yield return Page.FromValues(page.Values.Select(this.selector).ToList(), page.ContinuationToken, page.GetRawResponse()); + } + } + } + } +} diff --git a/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs b/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs index 759e6c95e..f4d8468cc 100644 --- a/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs +++ b/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs @@ -14,150 +14,98 @@ namespace DurableTask.AzureStorage.Storage { using System; - using System.Threading; - using System.Threading.Tasks; + using Azure.Core; + using Azure.Data.Tables; + using Azure.Storage.Blobs; + using Azure.Storage.Queues; + using DurableTask.AzureStorage.Http; using DurableTask.AzureStorage.Monitoring; - using Microsoft.WindowsAzure.Storage; - using Microsoft.WindowsAzure.Storage.Auth; - using Microsoft.WindowsAzure.Storage.Blob; - using Microsoft.WindowsAzure.Storage.Queue; - using Microsoft.WindowsAzure.Storage.Table; class AzureStorageClient { - static readonly TimeSpan StorageMaximumExecutionTime = TimeSpan.FromMinutes(2); - readonly CloudBlobClient blobClient; - readonly CloudQueueClient queueClient; - readonly CloudTableClient tableClient; - readonly SemaphoreSlim requestThrottleSemaphore; - - public AzureStorageClient(AzureStorageOrchestrationServiceSettings settings) : - this(settings.StorageAccountDetails == null ? - CloudStorageAccount.Parse(settings.StorageConnectionString) : settings.StorageAccountDetails.ToCloudStorageAccount(), - settings) - { } - - public AzureStorageClient(CloudStorageAccount account, AzureStorageOrchestrationServiceSettings settings) - { - this.Settings = settings; - - this.BlobAccountName = GetAccountName(account.Credentials, settings, account.BlobStorageUri, "blob"); - this.QueueAccountName = GetAccountName(account.Credentials, settings, account.QueueStorageUri, "queue"); - this.TableAccountName = GetAccountName(account.Credentials, settings, account.TableStorageUri, "table"); - this.Stats = new AzureStorageOrchestrationServiceStats(); - this.queueClient = account.CreateCloudQueueClient(); - this.queueClient.BufferManager = SimpleBufferManager.Shared; - this.blobClient = account.CreateCloudBlobClient(); - this.blobClient.BufferManager = SimpleBufferManager.Shared; + readonly BlobServiceClient blobClient; + readonly QueueServiceClient queueClient; + readonly TableServiceClient tableClient; - this.blobClient.DefaultRequestOptions.MaximumExecutionTime = StorageMaximumExecutionTime; - - if (settings.HasTrackingStoreStorageAccount) + public AzureStorageClient(AzureStorageOrchestrationServiceSettings settings) + { + if (settings == null) { - var trackingStoreAccount = settings.TrackingStoreStorageAccountDetails.ToCloudStorageAccount(); - this.tableClient = trackingStoreAccount.CreateCloudTableClient(); + throw new ArgumentNullException(nameof(settings)); } - else + + if (settings.StorageAccountClientProvider == null) { - this.tableClient = account.CreateCloudTableClient(); + throw new ArgumentException("Storage account client provider is not specified.", nameof(settings)); } - this.tableClient.BufferManager = SimpleBufferManager.Shared; + this.Settings = settings; + this.Stats = new AzureStorageOrchestrationServiceStats(); + + var throttlingPolicy = new ThrottlingHttpPipelinePolicy(this.Settings.MaxStorageOperationConcurrency); + var timeoutPolicy = new LeaseTimeoutHttpPipelinePolicy(this.Settings.LeaseRenewInterval); + var monitoringPolicy = new MonitoringHttpPipelinePolicy(this.Stats); - this.requestThrottleSemaphore = new SemaphoreSlim(this.Settings.MaxStorageOperationConcurrency); + this.blobClient = CreateClient(settings.StorageAccountClientProvider.Blob, ConfigureClientPolicies); + this.queueClient = CreateClient(settings.StorageAccountClientProvider.Queue, ConfigureClientPolicies); + this.tableClient = CreateClient( + settings.HasTrackingStoreStorageAccount + ? settings.TrackingServiceClientProvider! + : settings.StorageAccountClientProvider.Table, + ConfigureClientPolicies); + + void ConfigureClientPolicies(TClientOptions options) where TClientOptions : ClientOptions + { + options.AddPolicy(throttlingPolicy!, HttpPipelinePosition.PerCall); + options.AddPolicy(timeoutPolicy!, HttpPipelinePosition.PerCall); + options.AddPolicy(monitoringPolicy!, HttpPipelinePosition.PerRetry); + } } public AzureStorageOrchestrationServiceSettings Settings { get; } public AzureStorageOrchestrationServiceStats Stats { get; } - public string BlobAccountName { get; } + public string BlobAccountName => this.blobClient.AccountName; - public string QueueAccountName { get; } + public string QueueAccountName => this.queueClient.AccountName; - public string TableAccountName { get; } + public string TableAccountName => this.tableClient.AccountName; - public Blob GetBlobReference(string container, string blobName, string? blobDirectory = null) + public Blob GetBlobReference(string container, string blobName) { - NameValidator.ValidateBlobName(blobName); - return new Blob(this, this.blobClient, container, blobName, blobDirectory); + return new Blob(this.blobClient, container, blobName); } internal Blob GetBlobReference(Uri blobUri) { - return new Blob(this, this.blobClient, blobUri); + return new Blob(this.blobClient, blobUri); } public BlobContainer GetBlobContainerReference(string container) { - NameValidator.ValidateContainerName(container); return new BlobContainer(this, this.blobClient, container); } public Queue GetQueueReference(string queueName) { - NameValidator.ValidateQueueName(queueName); return new Queue(this, this.queueClient, queueName); } public Table GetTableReference(string tableName) { - NameValidator.ValidateTableName(tableName); return new Table(this, this.tableClient, tableName); } - public Task MakeBlobStorageRequest(Func> storageRequest, string operationName, string? clientRequestId = null) => - this.MakeStorageRequest(storageRequest, BlobAccountName, operationName, clientRequestId); - - public Task MakeQueueStorageRequest(Func> storageRequest, string operationName, string? clientRequestId = null) => - this.MakeStorageRequest(storageRequest, QueueAccountName, operationName, clientRequestId); - - public Task MakeTableStorageRequest(Func> storageRequest, string operationName, string? clientRequestId = null) => - this.MakeStorageRequest(storageRequest, TableAccountName, operationName, clientRequestId); - - public Task MakeBlobStorageRequest(Func storageRequest, string operationName, string? clientRequestId = null, bool force = false) => - this.MakeStorageRequest(storageRequest, BlobAccountName, operationName, clientRequestId, force); - - public Task MakeQueueStorageRequest(Func storageRequest, string operationName, string? clientRequestId = null) => - this.MakeStorageRequest(storageRequest, QueueAccountName, operationName, clientRequestId); - - public Task MakeTableStorageRequest(Func storageRequest, string operationName, string? clientRequestId = null) => - this.MakeStorageRequest(storageRequest, TableAccountName, operationName, clientRequestId); - - private async Task MakeStorageRequest(Func> storageRequest, string accountName, string operationName, string? clientRequestId = null, bool force = false) + static TClient CreateClient( + IStorageServiceClientProvider storageProvider, + Action configurePolicies) + where TClientOptions : ClientOptions { - if (!force) - { - await requestThrottleSemaphore.WaitAsync(); - } - - try - { - return await TimeoutHandler.ExecuteWithTimeout(operationName, accountName, this.Settings, storageRequest, this.Stats, clientRequestId); - } - catch (StorageException ex) - { - throw new DurableTaskStorageException(ex); - } - finally - { - if (!force) - { - requestThrottleSemaphore.Release(); - } - } - } - - private Task MakeStorageRequest(Func storageRequest, string accountName, string operationName, string? clientRequestId = null, bool force = false) => - this.MakeStorageRequest((context, cancellationToken) => WrapFunctionWithReturnType(storageRequest, context, cancellationToken), accountName, operationName, clientRequestId, force); + TClientOptions options = storageProvider.CreateOptions(); + configurePolicies?.Invoke(options); - private static async Task WrapFunctionWithReturnType(Func storageRequest, OperationContext context, CancellationToken cancellationToken) - { - await storageRequest(context, cancellationToken); - return null; + return storageProvider.CreateClient(options); } - - private static string GetAccountName(StorageCredentials credentials, AzureStorageOrchestrationServiceSettings settings, StorageUri serviceUri, string service) => - credentials.AccountName ?? settings.StorageAccountDetails?.AccountName ?? serviceUri.GetAccountName(service) ?? "(unknown)"; } } diff --git a/src/DurableTask.AzureStorage/Storage/Blob.cs b/src/DurableTask.AzureStorage/Storage/Blob.cs index f0ea31323..03f664fb0 100644 --- a/src/DurableTask.AzureStorage/Storage/Blob.cs +++ b/src/DurableTask.AzureStorage/Storage/Blob.cs @@ -15,127 +15,142 @@ namespace DurableTask.AzureStorage.Storage { using System; using System.IO; + using System.Linq; + using System.Text; + using System.Threading; using System.Threading.Tasks; - using Microsoft.WindowsAzure.Storage; - using Microsoft.WindowsAzure.Storage.Blob; + using Azure; + using Azure.Storage.Blobs; + using Azure.Storage.Blobs.Models; + using Azure.Storage.Blobs.Specialized; class Blob { - readonly AzureStorageClient azureStorageClient; - readonly CloudBlobClient blobClient; - readonly CloudBlockBlob cloudBlockBlob; + readonly BlockBlobClient blockBlobClient; - public Blob(AzureStorageClient azureStorageClient, CloudBlobClient blobClient, string containerName, string blobName, string? blobDirectory = null) + public Blob(BlobServiceClient blobServiceClient, string containerName, string blobName) { - this.azureStorageClient = azureStorageClient; - this.blobClient = blobClient; this.Name = blobName; - var fullBlobPath = blobDirectory != null ? Path.Combine(blobDirectory, this.Name) : blobName; - - this.cloudBlockBlob = this.blobClient.GetContainerReference(containerName).GetBlockBlobReference(fullBlobPath); + this.blockBlobClient = blobServiceClient.GetBlobContainerClient(containerName).GetBlockBlobClient(blobName); } - public Blob(AzureStorageClient azureStorageClient, CloudBlobClient blobClient, Uri blobUri) + public Blob(BlobServiceClient blobServiceClient, Uri blobUri) { - this.azureStorageClient = azureStorageClient; - this.blobClient = blobClient; - this.cloudBlockBlob = new CloudBlockBlob(blobUri, blobClient.Credentials); + string serviceString = blobServiceClient.Uri.AbsoluteUri; + if (serviceString[serviceString.Length - 1] != '/') + { + serviceString += '/'; + } + + string blobString = blobUri.AbsoluteUri; + if (blobString.IndexOf(serviceString, StringComparison.Ordinal) != 0) + { + throw new ArgumentException("Blob is not present in the storage account", nameof(blobUri)); + } + + // Create a relative URI by removing the service's address + // ie. // -> / + string remaining = blobString.Substring(serviceString.Length); + int containerEnd = remaining.IndexOf('/'); + if (containerEnd == -1) + { + throw new ArgumentException("Missing blob container", nameof(blobUri)); + } + + this.blockBlobClient = blobServiceClient + .GetBlobContainerClient(remaining.Substring(0, containerEnd)) + .GetBlockBlobClient(remaining.Substring(containerEnd + 1)); } public string? Name { get; } - public bool IsLeased => this.cloudBlockBlob.Properties.LeaseState == LeaseState.Leased; + public string AbsoluteUri => this.blockBlobClient.Uri.AbsoluteUri; - public string AbsoluteUri => this.cloudBlockBlob.Uri.AbsoluteUri; + public async Task ExistsAsync(CancellationToken cancellationToken = default) + { + return await this.blockBlobClient.ExistsAsync(cancellationToken).DecorateFailure(); + } - public async Task ExistsAsync() + public async Task DeleteIfExistsAsync(CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlockBlob.ExistsAsync(null, context, cancellationToken), - "Blob Exists"); + return await this.blockBlobClient.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, cancellationToken: cancellationToken).DecorateFailure(); } - public async Task DeleteIfExistsAsync() + public async Task IsLeasedAsync(CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlockBlob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, null, context, cancellationToken), - "Blob Delete"); + BlobProperties properties = await this.blockBlobClient.GetPropertiesAsync(cancellationToken: cancellationToken).DecorateFailure(); + return properties.LeaseState == LeaseState.Leased; } - public async Task UploadTextAsync(string content, string? leaseId = null, bool ifDoesntExist = false) + public async Task UploadTextAsync(string content, string? leaseId = null, bool ifDoesntExist = false, CancellationToken cancellationToken = default) { - AccessCondition? accessCondition = null; + BlobRequestConditions? conditions = null; if (ifDoesntExist) { - accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*"); + conditions = new BlobRequestConditions { IfNoneMatch = ETag.All }; } else if (leaseId != null) { - accessCondition = AccessCondition.GenerateLeaseCondition(leaseId); + conditions = new BlobRequestConditions { LeaseId = leaseId }; } - await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlockBlob.UploadTextAsync(content, null, accessCondition, null, context, cancellationToken), - "Blob UploadText"); + using var buffer = new MemoryStream(Encoding.UTF8.GetBytes(content), writable: false); + await this.blockBlobClient.UploadAsync(buffer, conditions: conditions, cancellationToken: cancellationToken).DecorateFailure(); } - public async Task UploadFromByteArrayAsync(byte[] buffer, int index, int byteCount) + public async Task UploadFromByteArrayAsync(byte[] buffer, int index, int byteCount, CancellationToken cancellationToken = default) { - await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlockBlob.UploadFromByteArrayAsync(buffer, index, byteCount, null, null, context, cancellationToken), - "Blob UploadFromByeArray"); + using var stream = new MemoryStream(buffer, index, byteCount, writable: false); + await this.blockBlobClient.UploadAsync(stream, cancellationToken: cancellationToken).DecorateFailure(); } - public async Task DownloadTextAsync() + public async Task DownloadTextAsync(CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlockBlob.DownloadTextAsync(null, null, null, context, cancellationToken), - "Blob DownloadText"); - } + BlobDownloadStreamingResult result = await this.blockBlobClient.DownloadStreamingAsync(cancellationToken: cancellationToken).DecorateFailure(); - public async Task DownloadToStreamAsync(MemoryStream target) - { - await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlockBlob.DownloadToStreamAsync(target, null, null, context, cancellationToken), - "Blob DownloadToStream"); + using var reader = new StreamReader(result.Content, Encoding.UTF8); + return await reader.ReadToEndAsync(); } - public async Task FetchAttributesAsync() + public Task DownloadToStreamAsync(MemoryStream target, CancellationToken cancellationToken = default) { - await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlockBlob.FetchAttributesAsync(null, null, context, cancellationToken), - "Blob FetchAttributes"); + return this.blockBlobClient.DownloadToAsync(target, cancellationToken: cancellationToken).DecorateFailure(); } - public async Task AcquireLeaseAsync(TimeSpan leaseInterval, string leaseId) + public async Task AcquireLeaseAsync(TimeSpan leaseInterval, string leaseId, CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlockBlob.AcquireLeaseAsync(leaseInterval, leaseId, null, null, context, cancellationToken), - "Blob AcquireLease"); - } + BlobLease lease = await this.blockBlobClient + .GetBlobLeaseClient(leaseId) + .AcquireAsync(leaseInterval, cancellationToken: cancellationToken) + .DecorateFailure(); + return lease.LeaseId; + } - public async Task ChangeLeaseAsync(string proposedLeaseId, string currentLeaseId) + public async Task ChangeLeaseAsync(string proposedLeaseId, string currentLeaseId, CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlockBlob.ChangeLeaseAsync(proposedLeaseId, accessCondition: AccessCondition.GenerateLeaseCondition(currentLeaseId), null, context, cancellationToken), - "Blob ChangeLease"); + BlobLease lease = await this.blockBlobClient + .GetBlobLeaseClient(currentLeaseId) + .ChangeAsync(proposedLeaseId, cancellationToken: cancellationToken) + .DecorateFailure(); + + return lease.LeaseId; } - public async Task RenewLeaseAsync(string leaseId) + public Task RenewLeaseAsync(string leaseId, CancellationToken cancellationToken = default) { - var requestOptions = new BlobRequestOptions { ServerTimeout = azureStorageClient.Settings.LeaseRenewInterval }; - await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlockBlob.RenewLeaseAsync(AccessCondition.GenerateLeaseCondition(leaseId), requestOptions, context, cancellationToken), - "Blob RenewLease", - force: true); // lease renewals should not be throttled + return this.blockBlobClient + .GetBlobLeaseClient(leaseId) + .RenewAsync(cancellationToken: cancellationToken) + .DecorateFailure(); } - public async Task ReleaseLeaseAsync(string leaseId) + public Task ReleaseLeaseAsync(string leaseId, CancellationToken cancellationToken = default) { - await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlockBlob.ReleaseLeaseAsync(AccessCondition.GenerateLeaseCondition(leaseId), null, context, cancellationToken), - "Blob ReleaseLease"); + return this.blockBlobClient + .GetBlobLeaseClient(leaseId) + .ReleaseAsync(cancellationToken: cancellationToken) + .DecorateFailure(); } } } diff --git a/src/DurableTask.AzureStorage/Storage/BlobContainer.cs b/src/DurableTask.AzureStorage/Storage/BlobContainer.cs index 6febf098b..175624db1 100644 --- a/src/DurableTask.AzureStorage/Storage/BlobContainer.cs +++ b/src/DurableTask.AzureStorage/Storage/BlobContainer.cs @@ -16,132 +16,91 @@ namespace DurableTask.AzureStorage.Storage using System; using System.Collections.Generic; using System.IO; + using System.Linq; using System.Threading; using System.Threading.Tasks; - using Microsoft.WindowsAzure.Storage; - using Microsoft.WindowsAzure.Storage.Blob; + using Azure; + using Azure.Storage.Blobs; + using Azure.Storage.Blobs.Models; + using Azure.Storage.Blobs.Specialized; class BlobContainer { readonly AzureStorageClient azureStorageClient; - readonly CloudBlobClient blobClient; readonly string containerName; - readonly CloudBlobContainer cloudBlobContainer; + readonly BlobContainerClient blobContainerClient; - public BlobContainer(AzureStorageClient azureStorageClient, CloudBlobClient blobClient, string name) + public BlobContainer(AzureStorageClient azureStorageClient, BlobServiceClient blobServiceClient, string name) { this.azureStorageClient = azureStorageClient; - this.blobClient = blobClient; this.containerName = name; - this.cloudBlobContainer = this.blobClient.GetContainerReference(this.containerName); + this.blobContainerClient = blobServiceClient.GetBlobContainerClient(this.containerName); } public Blob GetBlobReference(string blobName, string? blobPrefix = null) { - var fullBlobName = blobPrefix != null ? Path.Combine(blobPrefix, blobName) : blobName; + string fullBlobName = blobPrefix != null ? Path.Combine(blobPrefix, blobName) : blobName; return this.azureStorageClient.GetBlobReference(this.containerName, fullBlobName); } - public async Task CreateIfNotExistsAsync() + public async Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlobContainer.CreateIfNotExistsAsync(BlobContainerPublicAccessType.Off, null, context, cancellationToken), - "Create Container"); + // TODO: Any encryption scope? + // If we received null, then the response must have been a 409 (Conflict) and the container must already exist + Response response = await this.blobContainerClient.CreateIfNotExistsAsync(PublicAccessType.None, cancellationToken: cancellationToken).DecorateFailure(); + return response != null; } - public async Task ExistsAsync() + public async Task ExistsAsync(CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlobContainer.ExistsAsync(null, context, cancellationToken), - "Container Exists"); + return await this.blobContainerClient.ExistsAsync(cancellationToken).DecorateFailure(); } - public async Task DeleteIfExistsAsync(string? appLeaseId = null) + public async Task DeleteIfExistsAsync(string? appLeaseId = null, CancellationToken cancellationToken = default) { - AccessCondition? accessCondition = null; + BlobRequestConditions? conditions = null; if (appLeaseId != null) { - accessCondition = new AccessCondition() { LeaseId = appLeaseId }; + conditions = new BlobRequestConditions { LeaseId = appLeaseId }; } - return await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlobContainer.DeleteIfExistsAsync(accessCondition, null, context, cancellationToken), - "Delete Container"); + return await this.blobContainerClient.DeleteIfExistsAsync(conditions, cancellationToken).DecorateFailure(); } - public async Task> ListBlobsAsync(string? blobDirectory = null) + public AsyncPageable ListBlobsAsync(string? prefix = null, CancellationToken cancellationToken = default) { - BlobContinuationToken? continuationToken = null; - Func> listBlobsFunction; - if (blobDirectory != null) - { - var cloudBlobDirectory = this.cloudBlobContainer.GetDirectoryReference(blobDirectory); - - listBlobsFunction = (context, cancellationToken) => cloudBlobDirectory.ListBlobsSegmentedAsync( - useFlatBlobListing: true, - blobListingDetails: BlobListingDetails.Metadata, - maxResults: null, - currentToken: continuationToken, - options: null, - operationContext: context, - cancellationToken: cancellationToken); - } - else - { - listBlobsFunction = (context, cancellationToken) => this.cloudBlobContainer.ListBlobsSegmentedAsync( - null, - useFlatBlobListing: true, - blobListingDetails: BlobListingDetails.Metadata, - maxResults: null, - currentToken: continuationToken, - options: null, - operationContext: context, - cancellationToken: cancellationToken); - } - - var blobList = new List(); - do - { - BlobResultSegment segment = await this.azureStorageClient.MakeBlobStorageRequest(listBlobsFunction, "ListBlobs"); - - continuationToken = segment.ContinuationToken; - - foreach (IListBlobItem listBlobItem in segment.Results) - { - CloudBlockBlob cloudBlockBlob = (CloudBlockBlob)listBlobItem; - var blobName = cloudBlockBlob.Name; - Blob blob = this.GetBlobReference(blobName); - blobList.Add(blob); - } - } - while (continuationToken != null); - - return blobList; + return new AsyncPageableProjection( + this.blobContainerClient.GetBlobsAsync(BlobTraits.Metadata, BlobStates.None, prefix, cancellationToken), + x => this.GetBlobReference(x.Name)).DecorateFailure(); } - public async Task ChangeLeaseAsync(string proposedLeaseId, string currentLeaseId) + public async Task AcquireLeaseAsync(TimeSpan leaseInterval, string leaseId, CancellationToken cancellationToken = default) { - AccessCondition accessCondition = new AccessCondition() { LeaseId = currentLeaseId }; + BlobLease lease = await this.blobContainerClient + .GetBlobLeaseClient(leaseId) + .AcquireAsync(leaseInterval, cancellationToken: cancellationToken) + .DecorateFailure(); - return await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlobContainer.ChangeLeaseAsync(proposedLeaseId, accessCondition, null, context, cancellationToken), - "Container ChangeLease"); + return lease.LeaseId; } - public async Task AcquireLeaseAsync(TimeSpan leaseInterval, string proposedLeaseId) + public async Task ChangeLeaseAsync(string proposedLeaseId, string currentLeaseId, CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlobContainer.AcquireLeaseAsync(leaseInterval, proposedLeaseId, null, null, context, cancellationToken), - "Container AcquireLease"); + BlobLease lease = await this.blobContainerClient + .GetBlobLeaseClient(currentLeaseId) + .ChangeAsync(proposedLeaseId, cancellationToken: cancellationToken) + .DecorateFailure(); + + return lease.LeaseId; } - public async Task RenewLeaseAsync(string leaseId) + public Task RenewLeaseAsync(string leaseId, CancellationToken cancellationToken = default) { - AccessCondition accessCondition = new AccessCondition() { LeaseId = leaseId }; - await this.azureStorageClient.MakeBlobStorageRequest( - (context, cancellationToken) => this.cloudBlobContainer.RenewLeaseAsync(accessCondition, null, context, cancellationToken), - "Container RenewLease"); + return this.blobContainerClient + .GetBlobLeaseClient(leaseId) + .RenewAsync(cancellationToken: cancellationToken) + .DecorateFailure(); } } } diff --git a/src/DurableTask.AzureStorage/Storage/ClientResponseExtensions.cs b/src/DurableTask.AzureStorage/Storage/ClientResponseExtensions.cs new file mode 100644 index 000000000..a452629fc --- /dev/null +++ b/src/DurableTask.AzureStorage/Storage/ClientResponseExtensions.cs @@ -0,0 +1,97 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Storage +{ + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Azure; + + static class ClientResponseExtensions + { + public static AsyncPageable DecorateFailure(this AsyncPageable paginatedResponse) where T : notnull => + new DecoratedAsyncPageable(paginatedResponse); + + public static async Task DecorateFailure(this Task responseTask) + { + try + { + return await responseTask; + } + catch (RequestFailedException rfe) + { + throw new DurableTaskStorageException(rfe); + } + } + + public static async Task> DecorateFailure(this Task> responseTask) + { + try + { + return await responseTask; + } + catch (RequestFailedException rfe) + { + throw new DurableTaskStorageException(rfe); + } + } + + sealed class DecoratedAsyncPageable : AsyncPageable where T : notnull + { + readonly AsyncPageable source; + + public DecoratedAsyncPageable(AsyncPageable source) => + this.source = source; + + public override IAsyncEnumerable> AsPages(string? continuationToken = null, int? pageSizeHint = null) => + new DecoratedPageEnumerable(this.source.AsPages(continuationToken, pageSizeHint)); + + sealed class DecoratedPageEnumerable : IAsyncEnumerable> + { + readonly IAsyncEnumerable> source; + + public DecoratedPageEnumerable(IAsyncEnumerable> source) => + this.source = source; + + public IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default) => + new DecoratedPageEnumerator(this.source.GetAsyncEnumerator(cancellationToken)); + + sealed class DecoratedPageEnumerator : IAsyncEnumerator> + { + readonly IAsyncEnumerator> source; + + public DecoratedPageEnumerator(IAsyncEnumerator> source) => + this.source = source; + + public Page Current => this.source.Current; + + public ValueTask DisposeAsync() => + this.source.DisposeAsync(); + + public async ValueTask MoveNextAsync() + { + try + { + return await this.source.MoveNextAsync(); + } + catch (RequestFailedException rfe) + { + throw new DurableTaskStorageException(rfe); + } + } + } + } + } + } +} diff --git a/src/DurableTask.AzureStorage/Storage/DurableTaskStorageException.cs b/src/DurableTask.AzureStorage/Storage/DurableTaskStorageException.cs index 0086a6ca8..6c31e2287 100644 --- a/src/DurableTask.AzureStorage/Storage/DurableTaskStorageException.cs +++ b/src/DurableTask.AzureStorage/Storage/DurableTaskStorageException.cs @@ -14,8 +14,8 @@ namespace DurableTask.AzureStorage.Storage { using System; - using Microsoft.WindowsAzure.Storage; - using Microsoft.WindowsAzure.Storage.Blob.Protocol; + using Azure; + using Azure.Storage.Blobs.Models; [Serializable] class DurableTaskStorageException : Exception @@ -34,12 +34,11 @@ public DurableTaskStorageException(string message, Exception inner) { } - public DurableTaskStorageException(StorageException storageException) - : base(storageException.Message, storageException) + public DurableTaskStorageException(RequestFailedException requestFailedException) + : base(requestFailedException.Message, requestFailedException) { - this.HttpStatusCode = storageException.RequestInformation.HttpStatusCode; - StorageExtendedErrorInformation extendedErrorInfo = storageException.RequestInformation.ExtendedErrorInformation; - if (extendedErrorInfo?.ErrorCode == BlobErrorCodeStrings.LeaseLost) + this.HttpStatusCode = requestFailedException.Status; + if (requestFailedException?.ErrorCode == BlobErrorCode.LeaseLost) { LeaseLost = true; } diff --git a/src/DurableTask.AzureStorage/Storage/TableResultResponseInfo.cs b/src/DurableTask.AzureStorage/Storage/OperationContext.cs similarity index 72% rename from src/DurableTask.AzureStorage/Storage/TableResultResponseInfo.cs rename to src/DurableTask.AzureStorage/Storage/OperationContext.cs index c5017c33c..61ef72d6b 100644 --- a/src/DurableTask.AzureStorage/Storage/TableResultResponseInfo.cs +++ b/src/DurableTask.AzureStorage/Storage/OperationContext.cs @@ -13,15 +13,15 @@ #nullable enable namespace DurableTask.AzureStorage.Storage { - using System.Collections.Generic; - using Microsoft.WindowsAzure.Storage.Table; + using System; + using Azure.Core.Pipeline; - class TableResultResponseInfo + static class OperationContext { - public long ElapsedMilliseconds { get; set; } - - public int RequestCount { get; set; } - - public IList? TableResults { get; set; } + public static IDisposable CreateClientRequestScope(Guid? clientRequestId = null) + { + clientRequestId ??= Guid.NewGuid(); + return HttpPipeline.CreateClientRequestIdScope(clientRequestId.ToString()); + } } } diff --git a/src/DurableTask.AzureStorage/Storage/Queue.cs b/src/DurableTask.AzureStorage/Storage/Queue.cs index 57b51e0f3..e82f588ea 100644 --- a/src/DurableTask.AzureStorage/Storage/Queue.cs +++ b/src/DurableTask.AzureStorage/Storage/Queue.cs @@ -17,179 +17,122 @@ namespace DurableTask.AzureStorage.Storage using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; + using Azure; + using Azure.Storage.Queues; + using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Monitoring; - using Microsoft.WindowsAzure.Storage.Queue; class Queue { readonly AzureStorageClient azureStorageClient; - readonly CloudQueueClient queueClient; readonly AzureStorageOrchestrationServiceStats stats; - readonly CloudQueue cloudQueue; + readonly QueueClient queueClient; - public Queue(AzureStorageClient azureStorageClient, CloudQueueClient queueClient, string queueName) + public Queue(AzureStorageClient azureStorageClient, QueueServiceClient queueServiceClient, string queueName) { this.azureStorageClient = azureStorageClient; - this.queueClient = queueClient; this.stats = this.azureStorageClient.Stats; this.Name = queueName; - this.cloudQueue = this.queueClient.GetQueueReference(this.Name); + this.queueClient = queueServiceClient.GetQueueClient(this.Name); } public string Name { get; } - public Uri Uri => this.cloudQueue.Uri; + public Uri Uri => this.queueClient.Uri; - public int? ApproximateMessageCount => this.cloudQueue.ApproximateMessageCount; + public async Task GetApproximateMessagesCountAsync(CancellationToken cancellationToken = default) + { + QueueProperties properties = await this.queueClient.GetPropertiesAsync(cancellationToken).DecorateFailure(); + return properties.ApproximateMessagesCount; + } - public async Task AddMessageAsync(QueueMessage queueMessage, TimeSpan? visibilityDelay, Guid? clientRequestId = null) + public async Task AddMessageAsync(string message, TimeSpan? visibilityDelay, Guid? clientRequestId = null, CancellationToken cancellationToken = default) { - // Infinite time to live - TimeSpan? timeToLive = TimeSpan.FromSeconds(-1); -#if NET462 - // WindowsAzure.Storage 7.2.1 does not allow infinite time to live. Passing in null will default the time to live to 7 days. - timeToLive = null; -#endif - await this.azureStorageClient.MakeQueueStorageRequest( - (context, cancellationToken) => this.cloudQueue.AddMessageAsync( - queueMessage.CloudQueueMessage, - timeToLive, + using IDisposable scope = OperationContext.CreateClientRequestScope(clientRequestId); + await this.queueClient + .SendMessageAsync( + message, visibilityDelay, - null, - context), - "Queue AddMessage", - clientRequestId?.ToString()); + TimeSpan.FromSeconds(-1), // Infinite time to live + cancellationToken) + .DecorateFailure(); this.stats.MessagesSent.Increment(); } - public async Task UpdateMessageAsync(QueueMessage queueMessage, TimeSpan visibilityTimeout, Guid? clientRequestId = null) + public async Task UpdateMessageAsync(QueueMessage queueMessage, TimeSpan visibilityTimeout, Guid? clientRequestId = null, CancellationToken cancellationToken = default) { - await this.azureStorageClient.MakeQueueStorageRequest( - (context, cancellationToken) => this.cloudQueue.UpdateMessageAsync( - queueMessage.CloudQueueMessage, - visibilityTimeout, - MessageUpdateFields.Visibility, - null, - context), - "Queue UpdateMessage", - clientRequestId?.ToString()); + using IDisposable scope = OperationContext.CreateClientRequestScope(clientRequestId); + await this.queueClient + .UpdateMessageAsync( + queueMessage.MessageId, + queueMessage.PopReceipt, + visibilityTimeout: visibilityTimeout, + cancellationToken: cancellationToken) + .DecorateFailure(); this.stats.MessagesUpdated.Increment(); } - public async Task DeleteMessageAsync(QueueMessage queueMessage, Guid? clientRequestId = null) + public async Task DeleteMessageAsync(QueueMessage queueMessage, Guid? clientRequestId = null, CancellationToken cancellationToken = default) { - await this.azureStorageClient.MakeQueueStorageRequest( - (context, cancellationToken) => this.cloudQueue.DeleteMessageAsync( - queueMessage.CloudQueueMessage, - null, - context), - "Queue DeleteMessage", - clientRequestId?.ToString()); + using IDisposable scope = OperationContext.CreateClientRequestScope(clientRequestId); + await this.queueClient + .DeleteMessageAsync( + queueMessage.MessageId, + queueMessage.PopReceipt, + cancellationToken) + .DecorateFailure(); } - public async Task GetMessageAsync(TimeSpan visibilityTimeout, CancellationToken callerCancellationToken) + public async Task GetMessageAsync(TimeSpan visibilityTimeout, CancellationToken cancellationToken = default) { - var cloudQueueMessage = await this.azureStorageClient.MakeQueueStorageRequest( - async (context, timeoutCancellationToken) => - { - using (var finalLinkedCts = CancellationTokenSource.CreateLinkedTokenSource(callerCancellationToken, timeoutCancellationToken)) - { - return await this.cloudQueue.GetMessageAsync( - visibilityTimeout, - null, - context, - finalLinkedCts.Token); - } - }, - "Queue GetMessage"); - - if (cloudQueueMessage == null) + QueueMessage message = await this.queueClient.ReceiveMessageAsync(visibilityTimeout, cancellationToken).DecorateFailure(); + + if (message == null) { return null; } this.stats.MessagesRead.Increment(); - return new QueueMessage(cloudQueueMessage); + return message; } - public async Task ExistsAsync() + public async Task ExistsAsync(CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeQueueStorageRequest( - (context, cancellationToken) => this.cloudQueue.ExistsAsync(null, context, cancellationToken), - "Queue Exists"); + return await this.queueClient.ExistsAsync(cancellationToken).DecorateFailure(); } - public async Task CreateIfNotExistsAsync() + public async Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeQueueStorageRequest( - (context, cancellationToken) => this.cloudQueue.CreateIfNotExistsAsync(null, context, cancellationToken), - "Queue Create"); + // If we received null, then the response must have been a 409 (Conflict) and the queue must already exist + Response response = await this.queueClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken).DecorateFailure(); + return response != null; } - public async Task DeleteIfExistsAsync() + public async Task DeleteIfExistsAsync(CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeQueueStorageRequest( - (context, cancellationToken) => this.cloudQueue.DeleteIfExistsAsync(null, context, cancellationToken), - "Queue Delete"); + return await this.queueClient.DeleteIfExistsAsync(cancellationToken).DecorateFailure(); } - public async Task> GetMessagesAsync(int batchSize, TimeSpan visibilityTimeout, CancellationToken callerCancellationToken) + public async Task> GetMessagesAsync(int batchSize, TimeSpan visibilityTimeout, CancellationToken cancellationToken = default) { - var cloudQueueMessages = await this.azureStorageClient.MakeQueueStorageRequest>( - async (context, timeoutCancellationToken) => - { - using (var finalLinkedCts = CancellationTokenSource.CreateLinkedTokenSource(callerCancellationToken, timeoutCancellationToken)) - { - return await this.cloudQueue.GetMessagesAsync( - batchSize, - visibilityTimeout, - null, - context, - finalLinkedCts.Token); - } - }, - "Queue GetMessages"); - - var queueMessages = new List(); - foreach (CloudQueueMessage cloudQueueMessage in cloudQueueMessages) - { - queueMessages.Add(new QueueMessage(cloudQueueMessage)); - this.stats.MessagesRead.Increment(); - } - - return queueMessages; + QueueMessage[] messages = await this.queueClient.ReceiveMessagesAsync(batchSize, visibilityTimeout, cancellationToken).DecorateFailure(); + this.stats.MessagesRead.Increment(messages.Length); + return messages; } - public async Task FetchAttributesAsync() + public async Task> PeekMessagesAsync(int batchSize, CancellationToken cancellationToken = default) { - await this.azureStorageClient.MakeQueueStorageRequest( - (context, cancellationToken) => this.cloudQueue.FetchAttributesAsync(null, context, cancellationToken), - "Queue FetchAttributes"); - } - - public async Task> PeekMessagesAsync(int batchSize) - { - var cloudQueueMessages = await this.azureStorageClient.MakeQueueStorageRequest>( - (context, cancellationToken) => this.cloudQueue.PeekMessagesAsync(batchSize, null, context, cancellationToken), - "Queue PeekMessages"); - - var queueMessages = new List(); - foreach (CloudQueueMessage cloudQueueMessage in cloudQueueMessages) - { - queueMessages.Add(new QueueMessage(cloudQueueMessage)); - this.stats.MessagesRead.Increment(); - } - - return queueMessages; + PeekedMessage[] messages = await this.queueClient.PeekMessagesAsync(batchSize, cancellationToken).DecorateFailure(); + this.stats.MessagesRead.Increment(messages.Length); + return messages; } - public async Task PeekMessageAsync() + public async Task PeekMessageAsync(CancellationToken cancellationToken = default) { - var queueMessage = await this.cloudQueue.PeekMessageAsync(); - return queueMessage == null ? null : new QueueMessage(queueMessage); + return await this.queueClient.PeekMessageAsync(cancellationToken).DecorateFailure(); } } } diff --git a/src/DurableTask.AzureStorage/Storage/QueueMessage.cs b/src/DurableTask.AzureStorage/Storage/QueueMessage.cs deleted file mode 100644 index df8f166e8..000000000 --- a/src/DurableTask.AzureStorage/Storage/QueueMessage.cs +++ /dev/null @@ -1,49 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- -#nullable enable -namespace DurableTask.AzureStorage.Storage -{ - using System; - using Microsoft.WindowsAzure.Storage.Queue; - - class QueueMessage - { - public QueueMessage(CloudQueueMessage cloudQueueMessage) - { - this.CloudQueueMessage = cloudQueueMessage; - this.Message = this.CloudQueueMessage.AsString; - this.Id = this.CloudQueueMessage.Id; - } - - public QueueMessage(string message) - { - this.CloudQueueMessage = new CloudQueueMessage(message); - this.Message = this.CloudQueueMessage.AsString; - this.Id = this.CloudQueueMessage.Id; - } - - public CloudQueueMessage CloudQueueMessage { get; } - - public string Message { get; } - - public string Id { get; } - - public int DequeueCount => this.CloudQueueMessage.DequeueCount; - - public DateTimeOffset? InsertionTime => this.CloudQueueMessage.InsertionTime; - - public DateTimeOffset? NextVisibleTime => this.CloudQueueMessage.NextVisibleTime; - - public string PopReceipt => this.CloudQueueMessage.PopReceipt; - } -} diff --git a/src/DurableTask.AzureStorage/Storage/StorageUriExtensions.cs b/src/DurableTask.AzureStorage/Storage/StorageUriExtensions.cs deleted file mode 100644 index 55ebd3d46..000000000 --- a/src/DurableTask.AzureStorage/Storage/StorageUriExtensions.cs +++ /dev/null @@ -1,72 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- - -namespace DurableTask.AzureStorage.Storage -{ - using System; - using System.Collections.Generic; - using System.Net; - using Microsoft.WindowsAzure.Storage; - - internal static class StorageUriExtensions - { - // Note that much of this class is based on internal logic from the Azure Storage SDK - // Ports: https://github.com/Azure/azure-sdk-for-net/blob/c1c61f10b855ccdd97c790d92f26765e99fd15a8/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs#L599 - // Account Name Parse: https://github.com/Azure/azure-sdk-for-net/blob/4162f6fa2445b2127468b9cfd080f01c9da88eba/sdk/storage/Azure.Storage.Blobs/src/BlobUriBuilder.cs#L166 - // Utilities: https://github.com/Azure/azure-sdk-for-net/blob/4162f6fa2445b2127468b9cfd080f01c9da88eba/sdk/storage/Azure.Storage.Common/src/Shared/UriExtensions.cs#L16 - - private static readonly HashSet SasPorts = new HashSet { 10000, 10001, 10002, 10003, 10004, 10100, 10101, 10102, 10103, 10104, 11000, 11001, 11002, 11003, 11004, 11100, 11101, 11102, 11103, 11104 }; - - public static string GetAccountName(this StorageUri storageUri, string service) - { - if (storageUri == null) - throw new ArgumentNullException(nameof(storageUri)); - - if (service == null) - throw new ArgumentNullException(nameof(service)); - - // Note that the primary and secondary endpoints must share the same resource - Uri uri = storageUri.PrimaryUri; - - if (IsHostIPEndPointStyle(uri)) - { - // In some scenarios, like for Azurite, the service URI looks like ://:/ - string path = GetPath(uri); - int accountEndIndex = path.IndexOf("/", StringComparison.InvariantCulture); - return accountEndIndex == -1 ? path : path.Substring(0, accountEndIndex); - } - - return GetAccountNameFromDomain(uri.Host, service); - } - - private static string GetPath(Uri uri) => - uri.AbsolutePath[0] == '/' ? uri.AbsolutePath.Substring(1) : uri.AbsolutePath; - - private static bool IsHostIPEndPointStyle(Uri uri) => - (!string.IsNullOrEmpty(uri.Host) && uri.Host.IndexOf(".", StringComparison.InvariantCulture) >= 0 && IPAddress.TryParse(uri.Host, out _)) - || SasPorts.Contains(uri.Port); - - private static string GetAccountNameFromDomain(string host, string serviceSubDomain) - { - // Typically, Azure Storage Service URIs are formatted as ://.. - int accountEndIndex = host.IndexOf(".", StringComparison.InvariantCulture); - if (accountEndIndex >= 0) - { - int serviceStartIndex = host.IndexOf(serviceSubDomain, accountEndIndex, StringComparison.InvariantCulture); - return serviceStartIndex > -1 ? host.Substring(0, accountEndIndex) : null; - } - - return null; - } - } -} diff --git a/src/DurableTask.AzureStorage/Storage/Table.cs b/src/DurableTask.AzureStorage/Storage/Table.cs index dc5815c03..8cca7dc08 100644 --- a/src/DurableTask.AzureStorage/Storage/Table.cs +++ b/src/DurableTask.AzureStorage/Storage/Table.cs @@ -17,277 +17,161 @@ namespace DurableTask.AzureStorage.Storage using System.Collections.Generic; using System.Diagnostics; using System.Linq; - using System.Text; using System.Threading; using System.Threading.Tasks; + using Azure; + using Azure.Data.Tables; + using Azure.Data.Tables.Models; using DurableTask.AzureStorage.Monitoring; - using Microsoft.WindowsAzure.Storage.Table; - using Newtonsoft.Json; class Table { readonly AzureStorageClient azureStorageClient; - readonly CloudTableClient tableClient; readonly AzureStorageOrchestrationServiceStats stats; - readonly CloudTable cloudTable; + readonly TableServiceClient tableServiceClient; + readonly TableClient tableClient; - public Table(AzureStorageClient azureStorageClient, CloudTableClient tableClient, string tableName) + public Table(AzureStorageClient azureStorageClient, TableServiceClient tableServiceClient, string tableName) { this.azureStorageClient = azureStorageClient; - this.tableClient = tableClient; this.Name = tableName; this.stats = this.azureStorageClient.Stats; - this.cloudTable = this.tableClient.GetTableReference(this.Name); + this.tableServiceClient = tableServiceClient; + this.tableClient = tableServiceClient.GetTableClient(tableName); } public string Name { get; } - public Uri Uri => this.cloudTable.Uri; + internal Uri Uri => this.tableClient.Uri; - public async Task CreateIfNotExistsAsync() + public async Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeTableStorageRequest( - (context, cancellationToken) => this.cloudTable.CreateIfNotExistsAsync(null, context, cancellationToken), - "Table Create"); + // If we received null, then the response must have been a 409 (Conflict) and the table must already exist + Response response = await this.tableClient.CreateIfNotExistsAsync(cancellationToken).DecorateFailure(); + return response != null; } - public async Task DeleteIfExistsAsync() + public async Task DeleteIfExistsAsync(CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeTableStorageRequest( - (context, cancellationToken) => this.cloudTable.DeleteIfExistsAsync(null, context, cancellationToken), - "Table Delete"); + // If we received null, then the response must have been a 404 (NotFound) and the table must not exist + Response response = await this.tableClient.DeleteAsync(cancellationToken).DecorateFailure(); + return response != null; } - public async Task ExistsAsync() + public async Task ExistsAsync(CancellationToken cancellationToken = default) { - return await this.azureStorageClient.MakeTableStorageRequest( - (context, cancellationToken) => this.cloudTable.ExistsAsync(null, context, cancellationToken), - "Table Exists"); - } - - public async Task ReplaceAsync(DynamicTableEntity tableEntity) - { - TableOperation tableOperation = TableOperation.Replace(tableEntity); + // TODO: Re-evaluate the use of an "Exists" method as it was intentional omitted from the client API + List tables = await this.tableServiceClient + .QueryAsync(filter: $"TableName eq '{tableClient.Name}'", cancellationToken: cancellationToken) + .DecorateFailure() + .ToListAsync(cancellationToken); - await ExecuteAsync(tableOperation, "Replace"); + return tables.Count > 0; } - public async Task DeleteAsync(DynamicTableEntity tableEntity) + public async Task ReplaceAsync(T tableEntity, ETag ifMatch, CancellationToken cancellationToken = default) where T : ITableEntity { - TableOperation tableOperation = TableOperation.Delete(tableEntity); - - await ExecuteAsync(tableOperation, "Delete"); + await this.tableClient.UpdateEntityAsync(tableEntity, ifMatch, TableUpdateMode.Replace, cancellationToken).DecorateFailure(); + this.stats.TableEntitiesWritten.Increment(); } - public async Task InsertAsync(DynamicTableEntity tableEntity) + public async Task DeleteAsync(T tableEntity, ETag ifMatch = default, CancellationToken cancellationToken = default) where T : ITableEntity { - TableOperation tableOperation = TableOperation.Insert(tableEntity); - - await ExecuteAsync(tableOperation, "Insert"); + await this.tableClient.DeleteEntityAsync(tableEntity.PartitionKey, tableEntity.RowKey, ifMatch, cancellationToken).DecorateFailure(); + this.stats.TableEntitiesWritten.Increment(); } - public async Task MergeAsync(DynamicTableEntity tableEntity) + public async Task InsertAsync(T tableEntity, CancellationToken cancellationToken = default) where T : ITableEntity { - TableOperation tableOperation = TableOperation.Merge(tableEntity); - - await ExecuteAsync(tableOperation, "Merge"); + await this.tableClient.AddEntityAsync(tableEntity, cancellationToken).DecorateFailure(); + this.stats.TableEntitiesWritten.Increment(); } - public async Task InsertOrMergeAsync(DynamicTableEntity tableEntity) + public async Task MergeAsync(T tableEntity, ETag ifMatch, CancellationToken cancellationToken = default) where T : ITableEntity { - TableOperation tableOperation = TableOperation.InsertOrMerge(tableEntity); - - await ExecuteAsync(tableOperation, "InsertOrMerge"); + await this.tableClient.UpdateEntityAsync(tableEntity, ifMatch, TableUpdateMode.Merge, cancellationToken).DecorateFailure(); + this.stats.TableEntitiesWritten.Increment(); } - public async Task InsertOrReplaceAsync(DynamicTableEntity tableEntity) + public async Task InsertOrMergeAsync(T tableEntity, CancellationToken cancellationToken = default) where T : ITableEntity { - TableOperation tableOperation = TableOperation.InsertOrReplace(tableEntity); - - await ExecuteAsync(tableOperation, "InsertOrReplace"); + await this.tableClient.UpsertEntityAsync(tableEntity, TableUpdateMode.Merge, cancellationToken).DecorateFailure(); + this.stats.TableEntitiesWritten.Increment(); } - private async Task ExecuteAsync(TableOperation operation, string operationType) + public async Task InsertOrReplaceAsync(T tableEntity, CancellationToken cancellationToken = default) where T : ITableEntity { - var storageTableResult = await this.azureStorageClient.MakeTableStorageRequest( - (context, cancellationToken) => this.cloudTable.ExecuteAsync(operation, null, context, cancellationToken), - "Table Execute " + operationType); - + await this.tableClient.UpsertEntityAsync(tableEntity, TableUpdateMode.Replace, cancellationToken).DecorateFailure(); this.stats.TableEntitiesWritten.Increment(); } - public async Task DeleteBatchAsync(IList entityBatch) + public async Task DeleteBatchAsync(IEnumerable entityBatch, CancellationToken cancellationToken = default) where T : ITableEntity { - return await this.ExecuteBatchAsync(entityBatch, "Delete", (batch, item) => { batch.Delete(item); return batch; }); + return await this.ExecuteBatchAsync(entityBatch, item => new TableTransactionAction(TableTransactionActionType.Delete, item), cancellationToken: cancellationToken); } - public async Task InsertOrMergeBatchAsync(IList entityBatch) + public async Task InsertOrMergeBatchAsync(IEnumerable entityBatch, CancellationToken cancellationToken = default) where T : ITableEntity { - this.stats.TableEntitiesWritten.Increment(entityBatch.Count); - return await this.ExecuteBatchAsync(entityBatch, "InsertOrMerge", (batch, item) => { batch.InsertOrMerge(item); return batch; }); - } + TableTransactionResults results = await this.ExecuteBatchAsync(entityBatch, item => new TableTransactionAction(TableTransactionActionType.UpsertMerge, item), cancellationToken: cancellationToken); - private async Task ExecuteBatchAsync( - IList entityBatch, - string batchType, - Func batchOperation) - { - List results = new List(); - int requestCount = 0; - long elapsedMilliseconds = 0; - int pageOffset = 0; - while (pageOffset < entityBatch.Count) + if (results.Responses.Count > 0) { - List entitiesInBatch = entityBatch.Skip(pageOffset).Take(100).ToList(); - - var batch = new TableBatchOperation(); - foreach (DynamicTableEntity item in entitiesInBatch) - { - batch = batchOperation(batch, item); - } - - var batchResults = await this.ExecuteBatchAsync(batch, batchType); - - elapsedMilliseconds += batchResults.ElapsedMilliseconds; - requestCount += batchResults.RequestCount; - results.AddRange(batchResults.TableResults); - pageOffset += entitiesInBatch.Count; + this.stats.TableEntitiesWritten.Increment(results.Responses.Count); } - - return new TableResultResponseInfo - { - ElapsedMilliseconds = elapsedMilliseconds, - RequestCount = requestCount, - TableResults = results - }; + return results; } - public async Task ExecuteBatchAsync(TableBatchOperation batchOperation, string batchType) + async Task ExecuteBatchAsync( + IEnumerable entityBatch, + Func batchOperation, + int batchSize = 100, + CancellationToken cancellationToken = default) where T : ITableEntity { - var stopwatch = new Stopwatch(); - long elapsedMilliseconds = 0; - - var batchResults = await this.azureStorageClient.MakeTableStorageRequest( - (context, timeoutToken) => this.cloudTable.ExecuteBatchAsync(batchOperation, null, context, timeoutToken), - "Table BatchExecute " + batchType); - - stopwatch.Stop(); - elapsedMilliseconds += stopwatch.ElapsedMilliseconds; - - this.stats.TableEntitiesWritten.Increment(batchOperation.Count); - - return new TableResultResponseInfo + if (batchSize > 100) { - ElapsedMilliseconds = elapsedMilliseconds, - RequestCount = 1, - TableResults = batchResults - }; - } - - public async Task> ExecuteQueryAsync(TableQuery query, CancellationToken callerCancellationToken) where T : ITableEntity, new() - { - var results = new List(); - TableContinuationToken? tableContinuationToken = null; + throw new ArgumentOutOfRangeException(nameof(batchSize), "Table storage does not support batch sizes greater than 100."); + } - var stopwatch = new Stopwatch(); - int requestCount = 0; - long elapsedMilliseconds = 0; + var resultsBuilder = new TableTransactionResultsBuilder(); + var batch = new List(batchSize); - while (true) + foreach (T entity in entityBatch) { - stopwatch.Start(); - - TableQuerySegment segment = await this.azureStorageClient.MakeTableStorageRequest( - async (context, timeoutCancellationToken) => - { - using var finalLinkedCts = CancellationTokenSource.CreateLinkedTokenSource(callerCancellationToken, timeoutCancellationToken); - return await this.cloudTable.ExecuteQuerySegmentedAsync(query, tableContinuationToken, null, context, finalLinkedCts.Token); - }, - "Table ExecuteQuerySegmented"); - - stopwatch.Stop(); - elapsedMilliseconds += stopwatch.ElapsedMilliseconds; - this.stats.TableEntitiesRead.Increment(segment.Results.Count); - requestCount++; - - results.AddRange(segment); - if (query.TakeCount > 0 && results.Count >= query.TakeCount) + batch.Add(batchOperation(entity)); + if (batch.Count == batchSize) { - break; - } - - tableContinuationToken = segment.ContinuationToken; - if (tableContinuationToken == null || callerCancellationToken.IsCancellationRequested) - { - break; + resultsBuilder.Add(await this.ExecuteBatchAsync(batch, cancellationToken)); + batch.Clear(); } } - return new TableEntitiesResponseInfo + if (batch.Count > 0) { - ElapsedMilliseconds = elapsedMilliseconds, - RequestCount = requestCount, - ReturnedEntities = results, - }; - } + resultsBuilder.Add(await this.ExecuteBatchAsync(batch, cancellationToken)); + } - public async Task> ExecuteQueryAsync(TableQuery query) where T : ITableEntity, new() - { - return await this.ExecuteQueryAsync(query, CancellationToken.None); + return resultsBuilder.ToResults(); } - public virtual async Task> ExecuteQuerySegmentAsync( - TableQuery query, - CancellationToken callerCancellationToken, - string? continuationToken = null) - where T : ITableEntity, new() + public async Task ExecuteBatchAsync(IEnumerable batchOperation, CancellationToken cancellationToken = default) { - var results = new List(); - TableContinuationToken? tableContinuationToken = null; - - if (!string.IsNullOrEmpty(continuationToken)) - { - var tokenContent = Encoding.UTF8.GetString(Convert.FromBase64String(continuationToken)); - tableContinuationToken = Utils.DeserializeFromJson(tokenContent); - } - var stopwatch = new Stopwatch(); - long elapsedMilliseconds = 0; - stopwatch.Start(); - - var segment = await this.azureStorageClient.MakeTableStorageRequest( - async (context, timeoutCancellationToken) => - { - using (var finalLinkedCts = CancellationTokenSource.CreateLinkedTokenSource(callerCancellationToken, timeoutCancellationToken)) - { - return await this.cloudTable.ExecuteQuerySegmentedAsync(query, tableContinuationToken, null, context, finalLinkedCts.Token); - } - }, - "Table ExecuteQuerySegmented"); + Response> response = await this.tableClient.SubmitTransactionAsync(batchOperation, cancellationToken).DecorateFailure(); + IReadOnlyList batchResults = response.Value; stopwatch.Stop(); - elapsedMilliseconds += stopwatch.ElapsedMilliseconds; - this.stats.TableEntitiesRead.Increment(segment.Results.Count); - results.AddRange(segment); + this.stats.TableEntitiesWritten.Increment(batchResults.Count); - string? newContinuationToken = null; - if (segment.ContinuationToken != null) - { - string tokenJson = Utils.SerializeToJson(segment.ContinuationToken); - newContinuationToken = Convert.ToBase64String(Encoding.UTF8.GetBytes(tokenJson)); - } + return new TableTransactionResults(batchResults, stopwatch.Elapsed); + } - return new TableEntitiesResponseInfo - { - ElapsedMilliseconds = elapsedMilliseconds, - RequestCount = 1, - ReturnedEntities = results, - ContinuationToken = newContinuationToken, - }; + public TableQueryResponse ExecuteQueryAsync(string? filter = null, int? maxPerPage = null, IEnumerable? select = null, CancellationToken cancellationToken = default) where T : class, ITableEntity, new() + { + return new TableQueryResponse(this.tableClient.QueryAsync(filter, maxPerPage, select, cancellationToken).DecorateFailure()); } } } diff --git a/src/DurableTask.AzureStorage/Storage/TableQueryResponse.cs b/src/DurableTask.AzureStorage/Storage/TableQueryResponse.cs new file mode 100644 index 000000000..3c0171093 --- /dev/null +++ b/src/DurableTask.AzureStorage/Storage/TableQueryResponse.cs @@ -0,0 +1,57 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Storage +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Azure; + + class TableQueryResponse : IAsyncEnumerable where T : notnull + { + readonly AsyncPageable _query; + + public TableQueryResponse(AsyncPageable query) + { + this._query = query ?? throw new ArgumentNullException(nameof(query)); + } + + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + return this._query.GetAsyncEnumerator(cancellationToken); + } + + public async Task> GetResultsAsync(string? continuationToken = null, int? pageSizeHint = null, CancellationToken cancellationToken = default) + { + var sw = Stopwatch.StartNew(); + + int pages = 0; + var entities = new List(); + await foreach (Page page in this._query.AsPages(continuationToken, pageSizeHint).WithCancellation(cancellationToken)) + { + pages++; + entities.AddRange(page.Values); + } + + sw.Stop(); + return new TableQueryResults(entities, sw.Elapsed, pages); + } + + public static implicit operator AsyncPageable(TableQueryResponse response) => + response._query; + } +} diff --git a/src/DurableTask.AzureStorage/Storage/TableEntitiesResponseInfo.cs b/src/DurableTask.AzureStorage/Storage/TableQueryResults.cs similarity index 61% rename from src/DurableTask.AzureStorage/Storage/TableEntitiesResponseInfo.cs rename to src/DurableTask.AzureStorage/Storage/TableQueryResults.cs index 4296c9b97..c7e871f66 100644 --- a/src/DurableTask.AzureStorage/Storage/TableEntitiesResponseInfo.cs +++ b/src/DurableTask.AzureStorage/Storage/TableQueryResults.cs @@ -13,16 +13,24 @@ #nullable enable namespace DurableTask.AzureStorage.Storage { + using System; using System.Collections.Generic; - class TableEntitiesResponseInfo + sealed class TableQueryResults { - public long ElapsedMilliseconds { get; set; } + public TableQueryResults(IReadOnlyList entities, TimeSpan elapsed, int requestCount) + { + this.Entities = entities ?? throw new ArgumentNullException(nameof(entities)); + this.Elapsed = elapsed; + this.RequestCount = requestCount; + } - public int RequestCount { get; set; } + public IReadOnlyList Entities { get; } - public IList? ReturnedEntities { get; set; } + public TimeSpan Elapsed { get; } - public string? ContinuationToken { get; set; } + public int ElapsedMilliseconds => (int)this.Elapsed.TotalMilliseconds; + + public int RequestCount { get; } } } diff --git a/src/DurableTask.AzureStorage/Storage/TableTransactionResults.cs b/src/DurableTask.AzureStorage/Storage/TableTransactionResults.cs new file mode 100644 index 000000000..9a7a509d0 --- /dev/null +++ b/src/DurableTask.AzureStorage/Storage/TableTransactionResults.cs @@ -0,0 +1,37 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Storage +{ + using System; + using System.Collections.Generic; + using Azure; + + sealed class TableTransactionResults + { + public TableTransactionResults(IReadOnlyList responses, TimeSpan elapsed, int requestCount = 1) + { + this.Responses = responses ?? throw new ArgumentNullException(nameof(responses)); + this.Elapsed = elapsed; + this.RequestCount = requestCount; + } + + public IReadOnlyList Responses { get; } + + public TimeSpan Elapsed { get; } + + public int ElapsedMilliseconds => (int)this.Elapsed.TotalMilliseconds; + + public int RequestCount { get; } + } +} diff --git a/src/DurableTask.AzureStorage/Storage/TableTransactionResultsBuilder.cs b/src/DurableTask.AzureStorage/Storage/TableTransactionResultsBuilder.cs new file mode 100644 index 000000000..05b1312b5 --- /dev/null +++ b/src/DurableTask.AzureStorage/Storage/TableTransactionResultsBuilder.cs @@ -0,0 +1,45 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Storage +{ + using System; + using System.Collections.Generic; + using Azure; + + sealed class TableTransactionResultsBuilder + { + TimeSpan _elapsed; + int _requestCount; + List _responses = new List(); + + public TableTransactionResultsBuilder Add(TableTransactionResults batch) + { + if (batch == null) + { + throw new ArgumentNullException(nameof(batch)); + } + + this._responses.AddRange(batch.Responses); + this._elapsed += batch.Elapsed; + this._requestCount += batch.RequestCount; + + return this; + } + + public TableTransactionResults ToResults() + { + return new TableTransactionResults(this._responses, this._elapsed, this._requestCount); + } + } +} diff --git a/src/DurableTask.AzureStorage/StorageAccountClientProvider.cs b/src/DurableTask.AzureStorage/StorageAccountClientProvider.cs new file mode 100644 index 000000000..cb7a38baf --- /dev/null +++ b/src/DurableTask.AzureStorage/StorageAccountClientProvider.cs @@ -0,0 +1,118 @@ + +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage +{ + using System; + using Azure.Core; + using Azure.Data.Tables; + using Azure.Storage.Blobs; + using Azure.Storage.Queues; + + /// + /// Represents a client provider for the services exposed by an Azure Storage Account. + /// + public sealed class StorageAccountClientProvider + { + /// + /// Initializes a new instance of the class that returns + /// service clients using the given . + /// + /// An Azure Storage connection string. + /// + /// is or consists entirely of white space characters. + /// + public StorageAccountClientProvider(string connectionString) + : this( + StorageServiceClientProvider.ForBlob(connectionString), + StorageServiceClientProvider.ForQueue(connectionString), + StorageServiceClientProvider.ForTable(connectionString)) + { } + + /// + /// Initializes a new instance of the class that returns + /// service clients using the given and credential. + /// + /// An Azure Storage account name. + /// A token credential for accessing the service. + /// An Azure Blob Storage service client whose connection is based on the given . + /// + /// + /// is or consists entirely of white space characters. + /// + /// -or- + /// is . + /// + public StorageAccountClientProvider(string accountName, TokenCredential tokenCredential) + : this( + StorageServiceClientProvider.ForBlob(accountName, tokenCredential), + StorageServiceClientProvider.ForQueue(accountName, tokenCredential), + StorageServiceClientProvider.ForTable(accountName, tokenCredential)) + { } + + /// + /// Initializes a new instance of the class that returns + /// service clients using the given service URIs and credential. + /// + /// An Azure Blob Storage service URI. + /// An Azure Queue Storage service URI. + /// An Azure Table Storage service URI. + /// A token credential for accessing the service. + /// + /// , , + /// , or is . + /// + public StorageAccountClientProvider(Uri blobServiceUri, Uri queueServiceUri, Uri tableServiceUri, TokenCredential tokenCredential) + : this( + StorageServiceClientProvider.ForBlob(blobServiceUri, tokenCredential), + StorageServiceClientProvider.ForQueue(queueServiceUri, tokenCredential), + StorageServiceClientProvider.ForTable(tableServiceUri, tokenCredential)) + { } + + /// + /// Initializes a new instance of the class that returns + /// service clients using the given client providers. + /// + /// An Azure Blob Storage service client provider. + /// An Azure Queue Storage service client provider. + /// An Azure Table Storage service client provider. + /// + /// , , or is . + /// + public StorageAccountClientProvider( + IStorageServiceClientProvider blob, + IStorageServiceClientProvider queue, + IStorageServiceClientProvider table) + { + this.Blob = blob ?? throw new ArgumentNullException(nameof(blob)); + this.Queue = queue ?? throw new ArgumentNullException(nameof(queue)); + this.Table = table ?? throw new ArgumentNullException(nameof(table)); + } + + /// + /// Gets the client provider for Azure Blob Storage. + /// + public IStorageServiceClientProvider Blob { get; } + + /// + /// Gets the client provider for Azure Queue Storage. + /// + public IStorageServiceClientProvider Queue { get; } + + /// + /// Gets the client provider for Azure Table Storage. + /// + public IStorageServiceClientProvider Table { get; } + } +} diff --git a/src/DurableTask.AzureStorage/StorageAccountDetails.cs b/src/DurableTask.AzureStorage/StorageAccountDetails.cs deleted file mode 100644 index 4e1474012..000000000 --- a/src/DurableTask.AzureStorage/StorageAccountDetails.cs +++ /dev/null @@ -1,98 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- - -namespace DurableTask.AzureStorage -{ - using System; - using Microsoft.WindowsAzure.Storage; - using Microsoft.WindowsAzure.Storage.Auth; - - /// - /// Connection details of the Azure Storage account - /// - public sealed class StorageAccountDetails - { - /// - /// The storage account credentials - /// - public StorageCredentials StorageCredentials { get; set; } - - /// - /// The storage account name - /// - public string AccountName { get; set; } - - /// - /// The storage account endpoint suffix - /// - public string EndpointSuffix { get; set; } - - /// - /// The storage account connection string. - /// - /// - /// If specified, this value overrides any other settings. - /// - public string ConnectionString { get; set; } - - /// - /// The data plane URI for the blob service of the storage account. - /// - public Uri BlobServiceUri { get; set; } - - /// - /// The data plane URI for the queue service of the storage account. - /// - public Uri QueueServiceUri { get; set; } - - /// - /// The data plane URI for the table service of the storage account. - /// - public Uri TableServiceUri { get; set; } - - /// - /// Convert this to its equivalent . - /// - /// The corresponding instance. - public CloudStorageAccount ToCloudStorageAccount() - { - if (!string.IsNullOrEmpty(this.ConnectionString)) - { - return CloudStorageAccount.Parse(this.ConnectionString); - } - else if (this.BlobServiceUri != null || this.QueueServiceUri != null || this.TableServiceUri != null) - { - if (this.BlobServiceUri == null || this.QueueServiceUri == null || this.TableServiceUri == null) - { - throw new InvalidOperationException( - $"If at least one Azure Storage service URI is specified, {nameof(BlobServiceUri)}, {nameof(QueueServiceUri)}, and {nameof(TableServiceUri)} must all be provided."); - } - - return new CloudStorageAccount( - this.StorageCredentials, - this.BlobServiceUri, - this.QueueServiceUri, - this.TableServiceUri, - fileEndpoint: null); - } - else - { - return new CloudStorageAccount( - this.StorageCredentials, - this.AccountName, - this.EndpointSuffix, - useHttps: true); - } - } - } -} diff --git a/src/DurableTask.AzureStorage/StorageServiceClientProvider.cs b/src/DurableTask.AzureStorage/StorageServiceClientProvider.cs new file mode 100644 index 000000000..aa6b08fb3 --- /dev/null +++ b/src/DurableTask.AzureStorage/StorageServiceClientProvider.cs @@ -0,0 +1,262 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage +{ + using System; + using System.Globalization; + using Azure.Core; + using Azure.Data.Tables; + using Azure.Storage.Blobs; + using Azure.Storage.Queues; + + /// + /// Represents a set of methods for easily creating instances of type + /// . + /// + public static class StorageServiceClientProvider + { + #region Blob + + /// + /// Creates an for the Azure Blob Storage service. + /// + /// An Azure Storage connection string for the blob service. + /// An optional set of client options. + /// An Azure Blob Storage service client whose connection is based on the given . + /// + /// is or consists entirely of white space characters. + /// + public static IStorageServiceClientProvider ForBlob(string connectionString, BlobClientOptions? options = null) + { + if (string.IsNullOrEmpty(connectionString)) + { + throw new ArgumentNullException(nameof(connectionString)); + } + + return new DefaultStorageServiceClientProvider( + o => new BlobServiceClient(connectionString, o), + options ?? new BlobClientOptions()); + } + + /// + /// Creates an for the Azure Blob Storage service. + /// + /// An Azure Storage account name. + /// A token credential for accessing the service. + /// An optional set of client options. + /// An Azure Blob Storage service client whose connection is based on the given . + /// + /// + /// is or consists entirely of white space characters. + /// + /// -or- + /// is . + /// + public static IStorageServiceClientProvider ForBlob( + string accountName, + TokenCredential tokenCredential, + BlobClientOptions? options = null) + { + return ForBlob(CreateDefaultServiceUri(accountName, "blob"), tokenCredential, options); + } + + /// + /// Creates an for the Azure Blob Storage service. + /// + /// An Azure Blob Storage service URI. + /// A token credential for accessing the service. + /// An optional set of client options. + /// An Azure Blob Storage service client whose connection is based on the given . + /// + /// or is . + /// + public static IStorageServiceClientProvider ForBlob( + Uri serviceUri, + TokenCredential tokenCredential, + BlobClientOptions? options = null) + { + if (tokenCredential == null) + { + throw new ArgumentNullException(nameof(tokenCredential)); + } + + return new DefaultStorageServiceClientProvider( + o => new BlobServiceClient(serviceUri, tokenCredential, o), + options ?? new BlobClientOptions()); + } + + #endregion + + #region Queue + + /// + /// Creates an for the Azure Queue Storage service. + /// + /// An Azure Storage connection string for the queue service. + /// An optional set of client options. + /// An Azure Queue Storage service client whose connection is based on the given . + /// + /// is or consists entirely of white space characters. + /// + public static IStorageServiceClientProvider ForQueue(string connectionString, QueueClientOptions? options = null) + { + if (string.IsNullOrEmpty(connectionString)) + { + throw new ArgumentNullException(nameof(connectionString)); + } + + return new DefaultStorageServiceClientProvider( + o => new QueueServiceClient(connectionString, o), + options ?? new QueueClientOptions()); + } + + /// + /// Creates an for the Azure Queue Storage service. + /// + /// An Azure Storage account name. + /// A token credential for accessing the service. + /// An optional set of client options. + /// An Azure Queue Storage service client whose connection is based on the given . + /// + /// + /// is or consists entirely of white space characters. + /// + /// -or- + /// is . + /// + public static IStorageServiceClientProvider ForQueue( + string accountName, + TokenCredential tokenCredential, + QueueClientOptions? options = null) + { + return ForQueue(CreateDefaultServiceUri(accountName, "queue"), tokenCredential, options); + } + + /// + /// Creates an for the Azure Queue Storage service. + /// + /// An Azure Queue Storage service URI. + /// A token credential for accessing the service. + /// An optional set of client options. + /// An Azure Queue Storage service client whose connection is based on the given . + /// + /// or is . + /// + public static IStorageServiceClientProvider ForQueue( + Uri serviceUri, + TokenCredential tokenCredential, + QueueClientOptions? options = null) + { + if (tokenCredential == null) + { + throw new ArgumentNullException(nameof(tokenCredential)); + } + + return new DefaultStorageServiceClientProvider( + o => new QueueServiceClient(serviceUri, tokenCredential, o), + options ?? new QueueClientOptions()); + } + + #endregion + + #region Table + + /// + /// Creates an for the Azure Table Storage service. + /// + /// An Azure Storage connection string for the table service. + /// An optional set of client options. + /// An Azure Table Storage service client whose connection is based on the given . + /// + /// is or consists entirely of white space characters. + /// + public static IStorageServiceClientProvider ForTable(string connectionString, TableClientOptions? options = null) + { + if (string.IsNullOrEmpty(connectionString)) + { + throw new ArgumentNullException(nameof(connectionString)); + } + + return new DefaultStorageServiceClientProvider( + o => new TableServiceClient(connectionString, o), + options ?? new TableClientOptions()); + } + + /// + /// Creates an for the Azure Table Storage service. + /// + /// An Azure Storage account name. + /// A token credential for accessing the service. + /// An optional set of client options. + /// An Azure Table Storage service client whose connection is based on the given . + /// + /// + /// is or consists entirely of white space characters. + /// + /// -or- + /// is . + /// + public static IStorageServiceClientProvider ForTable( + string accountName, + TokenCredential tokenCredential, + TableClientOptions? options = null) + { + return ForTable(CreateDefaultServiceUri(accountName, "table"), tokenCredential, options); + } + + /// + /// Creates an for the Azure Table Storage service. + /// + /// An Azure Table Storage service URI. + /// A token credential for accessing the service. + /// An optional set of client options. + /// An Azure Table Storage service client whose connection is based on the given . + /// + /// or is . + /// + public static IStorageServiceClientProvider ForTable( + Uri serviceUri, + TokenCredential tokenCredential, + TableClientOptions? options = null) + { + if (tokenCredential == null) + { + throw new ArgumentNullException(nameof(tokenCredential)); + } + + return new DefaultStorageServiceClientProvider( + o => new TableServiceClient(serviceUri, tokenCredential, o), + options ?? new TableClientOptions()); + } + + #endregion + + static Uri CreateDefaultServiceUri(string accountName, string service) + { + if (string.IsNullOrWhiteSpace(accountName)) + { + throw new ArgumentNullException(nameof(accountName)); + } + + if (string.IsNullOrWhiteSpace(service)) + { + throw new ArgumentNullException(nameof(service)); + } + + return new Uri( + string.Format(CultureInfo.InvariantCulture, "https://{0}.{1}.core.windows.net/", accountName, service), + UriKind.Absolute); + } + } +} diff --git a/src/DurableTask.AzureStorage/TimeoutHandler.cs b/src/DurableTask.AzureStorage/TimeoutHandler.cs deleted file mode 100644 index bd65568cc..000000000 --- a/src/DurableTask.AzureStorage/TimeoutHandler.cs +++ /dev/null @@ -1,138 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- - -namespace DurableTask.AzureStorage -{ - using DurableTask.AzureStorage.Monitoring; - using Microsoft.WindowsAzure.Storage; - using System; - using System.Diagnostics; - using System.Threading; - using System.Threading.Tasks; - - // Class that acts as a timeout handler to wrap Azure Storage calls, mitigating a deadlock that occurs with Azure Storage SDK 9.3.3. - // The TimeoutHandler class is based off of the similar Azure Functions fix seen here: https://github.com/Azure/azure-webjobs-sdk/pull/2291 - internal static class TimeoutHandler - { - private static int NumTimeoutsHit = 0; - - private static DateTime LastTimeoutHit = DateTime.MinValue; - - /// - /// Process kill action. This is exposed here to allow override from tests. - /// - private static Action ProcessKillAction = (errorMessage) => Environment.FailFast(errorMessage); - - public static async Task ExecuteWithTimeout( - string operationName, - string account, - AzureStorageOrchestrationServiceSettings settings, - Func> operation, - AzureStorageOrchestrationServiceStats stats = null, - string clientRequestId = null) - { - OperationContext context = new OperationContext() { ClientRequestID = clientRequestId ?? Guid.NewGuid().ToString() }; - if (Debugger.IsAttached) - { - // ignore long delays while debugging - return await operation(context, CancellationToken.None); - } - - while (true) - { - using (var cts = new CancellationTokenSource()) - { - Task timeoutTask = Task.Delay(settings.StorageRequestsTimeout, cts.Token); - Task operationTask = operation(context, cts.Token); - - if (stats != null) - { - stats.StorageRequests.Increment(); - } - Task completedTask = await Task.WhenAny(timeoutTask, operationTask); - - if (Equals(timeoutTask, completedTask)) - { - // If more less than DefaultTimeoutCooldown passed, increase timeouts count - // otherwise, reset the count to 1, since this is the first timeout we receive - // after a long (enough) while - if (LastTimeoutHit + settings.StorageRequestsTimeoutCooldown > DateTime.UtcNow) - { - NumTimeoutsHit++; - } - else - { - NumTimeoutsHit = 1; - } - - LastTimeoutHit = DateTime.UtcNow; - - string taskHubName = settings?.TaskHubName; - if (NumTimeoutsHit < settings.MaxNumberOfTimeoutsBeforeRecycle) - { - string message = $"The operation '{operationName}' with id '{context.ClientRequestID}' did not complete in '{settings.StorageRequestsTimeout}'. Hit {NumTimeoutsHit} out of {settings.MaxNumberOfTimeoutsBeforeRecycle} allowed timeouts. Retrying the operation."; - settings.Logger.GeneralWarning(account ?? "", taskHubName ?? "", message); - - cts.Cancel(); - continue; - } - else - { - string message = $"The operation '{operationName}' with id '{context.ClientRequestID}' did not complete in '{settings.StorageRequestsTimeout}'. Hit maximum number ({settings.MaxNumberOfTimeoutsBeforeRecycle}) of timeouts. Terminating the process to mitigate potential deadlock."; - settings.Logger.GeneralError(account ?? "", taskHubName ?? "", message); - - // Delay to ensure the ETW event gets written - await Task.Delay(TimeSpan.FromSeconds(3)); - - bool executeFailFast = true; - Task gracefulShutdownTask = Task.Run(async () => - { - try - { - return await settings.OnImminentFailFast(message); - } - catch (Exception) - { - return true; - } - }); - - await Task.WhenAny(gracefulShutdownTask, Task.Delay(TimeSpan.FromSeconds(35))); - - if (gracefulShutdownTask.IsCompleted) - { - executeFailFast = gracefulShutdownTask.Result; - } - - if (executeFailFast) - { - TimeoutHandler.ProcessKillAction(message); - } - else - { - // Technically we don't need else as the action above would have killed the process. - // However tests don't kill the process so putting in else. - throw new TimeoutException(message); - } - } - - } - - cts.Cancel(); - - return await operationTask; - } - } - } - } -} diff --git a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs index 860aaa29e..f607c2c67 100644 --- a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs @@ -19,15 +19,17 @@ namespace DurableTask.AzureStorage.Tracking using System.Linq; using System.Net; using System.Reflection; + using System.Runtime.CompilerServices; using System.Runtime.Serialization; using System.Text; using System.Threading; using System.Threading.Tasks; + using Azure; + using Azure.Data.Tables; using DurableTask.AzureStorage.Monitoring; using DurableTask.AzureStorage.Storage; using DurableTask.Core; using DurableTask.Core.History; - using Microsoft.WindowsAzure.Storage.Table; /// /// Tracking store for use with . Uses Azure Tables and Azure Blobs to store runtime state. @@ -38,8 +40,9 @@ class AzureTableTrackingStore : TrackingStoreBase const string InputProperty = "Input"; const string ResultProperty = "Result"; const string OutputProperty = "Output"; - const string RowKeyProperty = "RowKey"; - const string PartitionKeyProperty = "PartitionKey"; + const string RowKeyProperty = nameof(ITableEntity.RowKey); + const string PartitionKeyProperty = nameof(ITableEntity.PartitionKey); + const string TimestampProperty = nameof(ITableEntity.Timestamp); const string SentinelRowKey = "sentinel"; const string IsCheckpointCompleteProperty = "IsCheckpointComplete"; const string CheckpointCompletedTimestampProperty = "CheckpointCompletedTimestamp"; @@ -65,7 +68,6 @@ class AzureTableTrackingStore : TrackingStoreBase readonly AzureStorageClient azureStorageClient; readonly AzureStorageOrchestrationServiceSettings settings; readonly AzureStorageOrchestrationServiceStats stats; - readonly TableEntityConverter tableEntityConverter; readonly IReadOnlyDictionary eventTypeMap; readonly MessageManager messageManager; @@ -77,7 +79,6 @@ class AzureTableTrackingStore : TrackingStoreBase this.messageManager = messageManager; this.settings = this.azureStorageClient.Settings; this.stats = this.azureStorageClient.Stats; - this.tableEntityConverter = new TableEntityConverter(); this.taskHubName = settings.TaskHubName; this.storageAccountName = this.azureStorageClient.TableAccountName; @@ -103,15 +104,16 @@ class AzureTableTrackingStore : TrackingStoreBase // For testing internal AzureTableTrackingStore( AzureStorageOrchestrationServiceStats stats, - Table instancesTable - ) + Table instancesTable) { this.stats = stats; this.InstancesTable = instancesTable; - this.settings = new AzureStorageOrchestrationServiceSettings(); - // Have to set FetchLargeMessageDataEnabled to false, as no MessageManager is - // instantiated for this test. - this.settings.FetchLargeMessageDataEnabled = false; + this.settings = new AzureStorageOrchestrationServiceSettings + { + // Have to set FetchLargeMessageDataEnabled to false, as no MessageManager is + // instantiated for this test. + FetchLargeMessageDataEnabled = false, + }; } internal Table HistoryTable { get; } @@ -119,56 +121,52 @@ Table instancesTable internal Table InstancesTable { get; } /// - public override Task CreateAsync() + public override Task CreateAsync(CancellationToken cancellationToken = default) { return Task.WhenAll(new Task[] { - this.HistoryTable.CreateIfNotExistsAsync(), - this.InstancesTable.CreateIfNotExistsAsync() + this.HistoryTable.CreateIfNotExistsAsync(cancellationToken), + this.InstancesTable.CreateIfNotExistsAsync(cancellationToken) }); } /// - public override Task DeleteAsync() + public override Task DeleteAsync(CancellationToken cancellationToken = default) { return Task.WhenAll(new Task[] { - this.HistoryTable.DeleteIfExistsAsync(), - this.InstancesTable.DeleteIfExistsAsync() + this.HistoryTable.DeleteIfExistsAsync(cancellationToken), + this.InstancesTable.DeleteIfExistsAsync(cancellationToken) }); } /// - public override async Task ExistsAsync() + public override async Task ExistsAsync(CancellationToken cancellationToken = default) { - return this.HistoryTable != null && this.InstancesTable != null && await this.HistoryTable.ExistsAsync() && await this.InstancesTable.ExistsAsync(); + return this.HistoryTable != null && this.InstancesTable != null && await this.HistoryTable.ExistsAsync(cancellationToken) && await this.InstancesTable.ExistsAsync(cancellationToken); } /// - public override async Task GetHistoryEventsAsync(string instanceId, string expectedExecutionId, CancellationToken cancellationToken = default(CancellationToken)) + public override async Task GetHistoryEventsAsync(string instanceId, string expectedExecutionId, CancellationToken cancellationToken = default) { - var historyEntitiesResponseInfo = await this.GetHistoryEntitiesResponseInfoAsync( - instanceId, - expectedExecutionId, - null, - cancellationToken); - - IList tableEntities = historyEntitiesResponseInfo.ReturnedEntities; + TableQueryResults results = await this + .GetHistoryEntitiesResponseInfoAsync(instanceId, expectedExecutionId, null, cancellationToken) + .GetResultsAsync(cancellationToken: cancellationToken); IList historyEvents; string executionId; - DynamicTableEntity sentinel = null; - if (tableEntities.Count > 0) + TableEntity sentinel = null; + if (results.Entities.Count > 0) { // The most recent generation will always be in the first history event. - executionId = tableEntities[0].Properties["ExecutionId"].StringValue; + executionId = results.Entities[0].GetString("ExecutionId"); // Convert the table entities into history events. - var events = new List(tableEntities.Count); + var events = new List(results.Entities.Count); - foreach (DynamicTableEntity entity in tableEntities) + foreach (TableEntity entity in results.Entities) { - if (entity.Properties["ExecutionId"].StringValue != executionId) + if (entity.GetString("ExecutionId") != executionId) { // The remaining entities are from a previous generation and can be discarded. break; @@ -183,16 +181,16 @@ public override async Task GetHistoryEventsAsync(string in } // Some entity properties may be stored in blob storage. - await this.DecompressLargeEntityProperties(entity); + await this.DecompressLargeEntityProperties(entity, cancellationToken); - events.Add((HistoryEvent)this.tableEntityConverter.ConvertFromTableEntity(entity, GetTypeForTableEntity)); + events.Add((HistoryEvent)TableEntityConverter.Deserialize(entity, GetTypeForTableEntity(entity))); } historyEvents = events; } else { - historyEvents = EmptyHistoryEventList; + historyEvents = Array.Empty(); executionId = expectedExecutionId; } @@ -200,12 +198,13 @@ public override async Task GetHistoryEventsAsync(string in // A sentinel won't exist only if no instance of this ID has ever existed or the instance history // was purged.The IsCheckpointCompleteProperty was newly added _after_ v1.6.4. DateTime checkpointCompletionTime = DateTime.MinValue; - sentinel = sentinel ?? tableEntities.LastOrDefault(e => e.RowKey == SentinelRowKey); - string eTagValue = sentinel?.ETag; + sentinel = sentinel ?? results.Entities.LastOrDefault(e => e.RowKey == SentinelRowKey); + ETag? eTagValue = sentinel?.ETag; if (sentinel != null && - sentinel.Properties.TryGetValue(CheckpointCompletedTimestampProperty, out EntityProperty timestampProperty)) + sentinel.TryGetValue(CheckpointCompletedTimestampProperty, out object timestampObj) && + timestampObj is DateTimeOffset timestampProperty) { - checkpointCompletionTime = timestampProperty.DateTime ?? DateTime.MinValue; + checkpointCompletionTime = timestampProperty.DateTime; } int currentEpisodeNumber = Utils.GetEpisodeNumber(historyEvents); @@ -217,51 +216,34 @@ public override async Task GetHistoryEventsAsync(string in executionId, historyEvents.Count, currentEpisodeNumber, - historyEntitiesResponseInfo.RequestCount, - historyEntitiesResponseInfo.ElapsedMilliseconds, - eTagValue, + results.RequestCount, + results.ElapsedMilliseconds, + eTagValue?.ToString(), checkpointCompletionTime); return new OrchestrationHistory(historyEvents, checkpointCompletionTime, eTagValue); } - async Task> GetHistoryEntitiesResponseInfoAsync(string instanceId, string expectedExecutionId, IList projectionColumns, CancellationToken cancellationToken = default(CancellationToken)) + TableQueryResponse GetHistoryEntitiesResponseInfoAsync(string instanceId, string expectedExecutionId, IList projectionColumns, CancellationToken cancellationToken) { - var sanitizedInstanceId = KeySanitation.EscapePartitionKey(instanceId); - string filterCondition = TableQuery.GenerateFilterCondition(PartitionKeyProperty, QueryComparisons.Equal, sanitizedInstanceId); + string filter = $"{nameof(ITableEntity.PartitionKey)} eq '{KeySanitation.EscapePartitionKey(instanceId)}'"; if (!string.IsNullOrEmpty(expectedExecutionId)) { - // Filter down to a specific generation. - var rowKeyOrExecutionId = TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, SentinelRowKey), - TableOperators.Or, - TableQuery.GenerateFilterCondition("ExecutionId", QueryComparisons.Equal, expectedExecutionId)); - - filterCondition = TableQuery.CombineFilters(filterCondition, TableOperators.And, rowKeyOrExecutionId); + filter += $" and ({nameof(ITableEntity.RowKey)} eq '{SentinelRowKey}' or {nameof(OrchestrationInstance.ExecutionId)} eq '{expectedExecutionId}')"; } - TableQuery query = new TableQuery().Where(filterCondition); - - if (projectionColumns != null) - { - query.Select(projectionColumns); - } - - var tableEntitiesResponseInfo = await this.HistoryTable.ExecuteQueryAsync(query, cancellationToken); - - return tableEntitiesResponseInfo; + return this.HistoryTable.ExecuteQueryAsync(filter, select: projectionColumns, cancellationToken: cancellationToken); } - async Task> QueryHistoryAsync(TableQuery query, string instanceId, CancellationToken cancellationToken) + async Task> QueryHistoryAsync(string filter, string instanceId, CancellationToken cancellationToken) { - var tableEntitiesResponseInfo = await this.HistoryTable.ExecuteQueryAsync(query, cancellationToken); + TableQueryResults results = await this + .HistoryTable.ExecuteQueryAsync(filter, cancellationToken: cancellationToken) + .GetResultsAsync(cancellationToken: cancellationToken); - var entities = tableEntitiesResponseInfo.ReturnedEntities; - - string executionId = entities.Count > 0 && entities.First().Properties.ContainsKey("ExecutionId") ? - entities[0].Properties["ExecutionId"].StringValue : - string.Empty; + IReadOnlyList entities = results.Entities; + string executionId = entities.FirstOrDefault()?.GetString(nameof(OrchestrationInstance.ExecutionId)) ?? string.Empty; this.settings.Logger.FetchedInstanceHistory( this.storageAccountName, this.taskHubName, @@ -269,15 +251,15 @@ async Task> QueryHistoryAsync(TableQuery> RewindHistoryAsync(string instanceId, IList failedLeaves, CancellationToken cancellationToken) + public override async IAsyncEnumerable RewindHistoryAsync(string instanceId, [EnumeratorCancellation] CancellationToken cancellationToken = default) { ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // REWIND ALGORITHM: @@ -289,45 +271,32 @@ public override async Task> RewindHistoryAsync(string instanceId, //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// bool hasFailedSubOrchestrations = false; + string partitionFilter = $"{nameof(ITableEntity.PartitionKey)} eq '{KeySanitation.EscapePartitionKey(instanceId)}'"; - string sanitizedInstanceId = KeySanitation.EscapePartitionKey(instanceId); - var partitionFilter = TableQuery.GenerateFilterCondition(PartitionKeyProperty, QueryComparisons.Equal, sanitizedInstanceId); - - var orchestratorStartedFilterCondition = partitionFilter; - var orchestratorStartedEventFilter = TableQuery.GenerateFilterCondition("EventType", QueryComparisons.Equal, nameof(EventType.OrchestratorStarted)); - orchestratorStartedFilterCondition = TableQuery.CombineFilters(orchestratorStartedFilterCondition, TableOperators.And, orchestratorStartedEventFilter); - - var orchestratorStartedQuery = new TableQuery().Where(orchestratorStartedFilterCondition); - - var orchestratorStartedEntities = await this.QueryHistoryAsync(orchestratorStartedQuery, instanceId, cancellationToken); + string orchestratorStartedFilter = $"{partitionFilter} and {nameof(HistoryEvent.EventType)} eq '{nameof(EventType.OrchestratorStarted)}'"; + IReadOnlyList orchestratorStartedEntities = await this.QueryHistoryAsync(orchestratorStartedFilter, instanceId, cancellationToken); // get most recent orchestratorStarted event - var recentStartRowKey = orchestratorStartedEntities.Max(x => x.RowKey); + string recentStartRowKey = orchestratorStartedEntities.Max(x => x.RowKey); var recentStartRow = orchestratorStartedEntities.Where(y => y.RowKey == recentStartRowKey).ToList(); - var executionId = recentStartRow[0].Properties["ExecutionId"].StringValue; - var instanceTimestamp = recentStartRow[0].Timestamp.DateTime; - - var rowsToUpdateFilterCondition = partitionFilter; - var executionIdFilter = TableQuery.GenerateFilterCondition("ExecutionId", QueryComparisons.Equal, executionId); - rowsToUpdateFilterCondition = TableQuery.CombineFilters(rowsToUpdateFilterCondition, TableOperators.And, executionIdFilter); - - var orchestrationStatusFilter = TableQuery.GenerateFilterCondition("OrchestrationStatus", QueryComparisons.Equal, "Failed"); - var failedEventFilter = TableQuery.GenerateFilterCondition("EventType", QueryComparisons.Equal, nameof(EventType.TaskFailed)); - var failedSubOrchestrationEventFilter = TableQuery.GenerateFilterCondition("EventType", QueryComparisons.Equal, nameof(EventType.SubOrchestrationInstanceFailed)); - - var failedQuerySegment = orchestrationStatusFilter; - failedQuerySegment = TableQuery.CombineFilters(failedQuerySegment, TableOperators.Or, failedEventFilter); - failedQuerySegment = TableQuery.CombineFilters(failedQuerySegment, TableOperators.Or, failedSubOrchestrationEventFilter); - - rowsToUpdateFilterCondition = TableQuery.CombineFilters(rowsToUpdateFilterCondition, TableOperators.And, failedQuerySegment); - - var rowsToUpdateQuery = new TableQuery().Where(rowsToUpdateFilterCondition); - - var entitiesToClear = await this.QueryHistoryAsync(rowsToUpdateQuery, instanceId, cancellationToken); - - foreach (DynamicTableEntity entity in entitiesToClear) + string executionId = recentStartRow[0].GetString(nameof(OrchestrationInstance.ExecutionId)); + DateTime instanceTimestamp = recentStartRow[0].Timestamp.GetValueOrDefault().DateTime; + + string executionIdFilter = $"{nameof(OrchestrationInstance.ExecutionId)} eq '{executionId}'"; + + var updateFilterBuilder = new StringBuilder(); + updateFilterBuilder.Append($"{partitionFilter}"); + updateFilterBuilder.Append($" and {executionIdFilter}"); + updateFilterBuilder.Append(" and ("); + updateFilterBuilder.Append($"{nameof(ExecutionCompletedEvent.OrchestrationStatus)} eq '{nameof(OrchestrationStatus.Failed)}'"); + updateFilterBuilder.Append($" or {nameof(HistoryEvent.EventType)} eq '{nameof(EventType.TaskFailed)}'"); + updateFilterBuilder.Append($" or {nameof(HistoryEvent.EventType)} eq '{nameof(EventType.SubOrchestrationInstanceFailed)}'"); + updateFilterBuilder.Append(')'); + + IReadOnlyList entitiesToClear = await this.QueryHistoryAsync(updateFilterBuilder.ToString(), instanceId, cancellationToken); + foreach (TableEntity entity in entitiesToClear) { - if (entity.Properties["ExecutionId"].StringValue != executionId) + if (entity.GetString(nameof(OrchestrationInstance.ExecutionId)) != executionId) { // the remaining entities are from a previous generation and can be discarded. break; @@ -338,97 +307,88 @@ public override async Task> RewindHistoryAsync(string instanceId, continue; } - // delete TaskScheduled corresponding to TaskFailed event - if (entity.Properties["EventType"].StringValue == nameof(EventType.TaskFailed)) - { - var taskScheduledId = entity.Properties["TaskScheduledId"].Int32Value.Value; - - var taskScheduledFilterCondition = partitionFilter; - taskScheduledFilterCondition = TableQuery.CombineFilters(taskScheduledFilterCondition, TableOperators.And, executionIdFilter); - - var eventIdFilter = TableQuery.GenerateFilterConditionForInt("EventId", QueryComparisons.Equal, taskScheduledId); - taskScheduledFilterCondition = TableQuery.CombineFilters(taskScheduledFilterCondition, TableOperators.And, eventIdFilter); - var taskScheduledEventFilter = TableQuery.GenerateFilterCondition("EventType", QueryComparisons.Equal, nameof(EventType.TaskScheduled)); - taskScheduledFilterCondition = TableQuery.CombineFilters(taskScheduledFilterCondition, TableOperators.And, taskScheduledEventFilter); - - var taskScheduledQuery = new TableQuery().Where(taskScheduledFilterCondition); + int? taskScheduledId = entity.GetInt32(nameof(TaskCompletedEvent.TaskScheduledId)); - var taskScheduledEntities = await QueryHistoryAsync(taskScheduledQuery, instanceId, cancellationToken); + var eventFilterBuilder = new StringBuilder(); + eventFilterBuilder.Append($"{partitionFilter}"); + eventFilterBuilder.Append($" and {executionIdFilter}"); + eventFilterBuilder.Append($" and {nameof(HistoryEvent.EventId)} eq {taskScheduledId.GetValueOrDefault()}"); - taskScheduledEntities[0].Properties["Reason"] = new EntityProperty("Rewound: " + taskScheduledEntities[0].Properties["EventType"].StringValue); - taskScheduledEntities[0].Properties["EventType"] = new EntityProperty(nameof(EventType.GenericEvent)); - - await this.HistoryTable.ReplaceAsync(taskScheduledEntities[0]); - } - - // delete SubOrchestratorCreated corresponding to SubOrchestraionInstanceFailed event - if (entity.Properties["EventType"].StringValue == nameof(EventType.SubOrchestrationInstanceFailed)) + switch (entity.GetString(nameof(HistoryEvent.EventType))) { - hasFailedSubOrchestrations = true; - var subOrchestrationId = entity.Properties["TaskScheduledId"].Int32Value.Value; - - var subOrchestratorCreatedFilterCondition = partitionFilter; - subOrchestratorCreatedFilterCondition = TableQuery.CombineFilters(subOrchestratorCreatedFilterCondition, TableOperators.And, executionIdFilter); - - var eventIdFilter = TableQuery.GenerateFilterConditionForInt("EventId", QueryComparisons.Equal, subOrchestrationId); - subOrchestratorCreatedFilterCondition = TableQuery.CombineFilters(subOrchestratorCreatedFilterCondition, TableOperators.And, eventIdFilter); - var subOrchestrationCreatedFilter = TableQuery.GenerateFilterCondition("EventType", QueryComparisons.Equal, nameof(EventType.SubOrchestrationInstanceCreated)); - subOrchestratorCreatedFilterCondition = TableQuery.CombineFilters(subOrchestratorCreatedFilterCondition, TableOperators.And, subOrchestrationCreatedFilter); - - var subOrchestratorCreatedQuery = new TableQuery().Where(subOrchestratorCreatedFilterCondition); + // delete TaskScheduled corresponding to TaskFailed event + case nameof(EventType.TaskFailed): + eventFilterBuilder.Append($" and {nameof(HistoryEvent.EventType)} eq '{nameof(EventType.TaskScheduled)}'"); + IReadOnlyList taskScheduledEntities = await this.QueryHistoryAsync(eventFilterBuilder.ToString(), instanceId, cancellationToken); + + TableEntity tsEntity = taskScheduledEntities[0]; + tsEntity[nameof(TaskFailedEvent.Reason)] = "Rewound: " + tsEntity.GetString(nameof(HistoryEvent.EventType)); + tsEntity[nameof(TaskFailedEvent.EventType)] = nameof(EventType.GenericEvent); + await this.HistoryTable.ReplaceAsync(tsEntity, tsEntity.ETag, cancellationToken); + break; - var subOrchesratrationEntities = await QueryHistoryAsync(subOrchestratorCreatedQuery, instanceId, cancellationToken); + // delete SubOrchestratorCreated corresponding to SubOrchestraionInstanceFailed event + case nameof(EventType.SubOrchestrationInstanceFailed): + hasFailedSubOrchestrations = true; - var soInstanceId = subOrchesratrationEntities[0].Properties["InstanceId"].StringValue; + eventFilterBuilder.Append($" and {nameof(HistoryEvent.EventType)} eq '{nameof(EventType.SubOrchestrationInstanceCreated)}'"); + IReadOnlyList subOrchesratrationEntities = await this.QueryHistoryAsync(eventFilterBuilder.ToString(), instanceId, cancellationToken); - // the SubORchestrationCreatedEvent is still healthy and will not be overwritten, just marked as rewound - subOrchesratrationEntities[0].Properties["Reason"] = new EntityProperty("Rewound: " + subOrchesratrationEntities[0].Properties["EventType"].StringValue); + // the SubOrchestrationCreatedEvent is still healthy and will not be overwritten, just marked as rewound + TableEntity soEntity = subOrchesratrationEntities[0]; + soEntity[nameof(SubOrchestrationInstanceFailedEvent.Reason)] = "Rewound: " + soEntity.GetString(nameof(HistoryEvent.EventType)); + await this.HistoryTable.ReplaceAsync(soEntity, soEntity.ETag, cancellationToken); - await this.HistoryTable.ReplaceAsync(subOrchesratrationEntities[0]); + // recursive call to clear out failure events on child instances + await foreach (string childInstanceId in this.RewindHistoryAsync(soEntity.GetString(nameof(OrchestrationInstance.InstanceId)), cancellationToken)) + { + yield return childInstanceId; + } - // recursive call to clear out failure events on child instances - await this.RewindHistoryAsync(soInstanceId, failedLeaves, cancellationToken); + break; } // "clear" failure event by making RewindEvent: replay ignores row while dummy event preserves rowKey - entity.Properties["Reason"] = new EntityProperty("Rewound: " + entity.Properties["EventType"].StringValue); - entity.Properties["EventType"] = new EntityProperty(nameof(EventType.GenericEvent)); + entity[nameof(TaskFailedEvent.Reason)] = "Rewound: " + entity.GetString(nameof(HistoryEvent.EventType)); + entity[nameof(TaskFailedEvent.EventType)] = nameof(EventType.GenericEvent); - await this.HistoryTable.ReplaceAsync(entity); + await this.HistoryTable.ReplaceAsync(entity, entity.ETag, cancellationToken); } // reset orchestration status in instance store table - await UpdateStatusForRewindAsync(instanceId); + await this.UpdateStatusForRewindAsync(instanceId, cancellationToken); if (!hasFailedSubOrchestrations) { - failedLeaves.Add(instanceId); + yield return instanceId; } - - return failedLeaves; } /// - public override async Task> GetStateAsync(string instanceId, bool allExecutions, bool fetchInput) + public override async IAsyncEnumerable GetStateAsync(string instanceId, bool allExecutions, bool fetchInput, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - return new[] { await this.GetStateAsync(instanceId, executionId: null, fetchInput: fetchInput) }; + InstanceStatus instanceStatus = await this.FetchInstanceStatusInternalAsync(instanceId, fetchInput, cancellationToken); + if (instanceStatus != null) + { + yield return instanceStatus.State; + } } #nullable enable /// - public override async Task GetStateAsync(string instanceId, string executionId, bool fetchInput) + public override async Task GetStateAsync(string instanceId, string executionId, bool fetchInput, CancellationToken cancellationToken = default) { - InstanceStatus? instanceStatus = await this.FetchInstanceStatusInternalAsync(instanceId, fetchInput); + InstanceStatus? instanceStatus = await this.FetchInstanceStatusInternalAsync(instanceId, fetchInput, cancellationToken); return instanceStatus?.State; } /// - public override Task FetchInstanceStatusAsync(string instanceId) + public override Task FetchInstanceStatusAsync(string instanceId, CancellationToken cancellationToken = default) { - return this.FetchInstanceStatusInternalAsync(instanceId, fetchInput: false); + return this.FetchInstanceStatusInternalAsync(instanceId, fetchInput: false, cancellationToken); } /// - async Task FetchInstanceStatusInternalAsync(string instanceId, bool fetchInput) + async Task FetchInstanceStatusInternalAsync(string instanceId, bool fetchInput, CancellationToken cancellationToken) { if (instanceId == null) { @@ -441,15 +401,17 @@ public override async Task> GetStateAsync(string insta FetchInput = fetchInput, }; - var tableEntitiesResponseInfo = await this.InstancesTable.ExecuteQueryAsync(queryCondition.ToTableQuery()); + ODataCondition odata = queryCondition.ToOData(); - var tableEntity = tableEntitiesResponseInfo.ReturnedEntities.FirstOrDefault(); + var sw = Stopwatch.StartNew(); - OrchestrationState? orchestrationState = null; - if (tableEntity != null) - { - orchestrationState = await this.ConvertFromAsync(tableEntity); - } + OrchestrationInstanceStatus? tableEntity = await this.InstancesTable + .ExecuteQueryAsync(odata.Filter, 1, odata.Select, cancellationToken) + .FirstOrDefaultAsync(); + + sw.Stop(); + + OrchestrationState? orchestrationState = tableEntity != null ? await this.ConvertFromAsync(tableEntity, cancellationToken) : null; this.settings.Logger.FetchedInstanceStatus( this.storageAccountName, @@ -457,7 +419,7 @@ public override async Task> GetStateAsync(string insta instanceId, orchestrationState?.OrchestrationInstance.ExecutionId ?? string.Empty, orchestrationState?.OrchestrationStatus.ToString() ?? "NotFound", - tableEntitiesResponseInfo.ElapsedMilliseconds); + sw.ElapsedMilliseconds); if (tableEntity == null || orchestrationState == null) { @@ -467,47 +429,13 @@ public override async Task> GetStateAsync(string insta return new InstanceStatus(orchestrationState, tableEntity.ETag); } #nullable disable - Task ConvertFromAsync(DynamicTableEntity tableEntity) + Task ConvertFromAsync(OrchestrationInstanceStatus tableEntity, CancellationToken cancellationToken) { - var properties = tableEntity.Properties; - var orchestrationInstanceStatus = ConvertFromAsync(properties); var instanceId = KeySanitation.UnescapePartitionKey(tableEntity.PartitionKey); - return ConvertFromAsync(orchestrationInstanceStatus, instanceId); - } - - static OrchestrationInstanceStatus ConvertFromAsync(IDictionary properties) - { - var orchestrationInstanceStatus = new OrchestrationInstanceStatus(); - - var type = typeof(OrchestrationInstanceStatus); - foreach (var pair in properties) - { - var property = type.GetProperty(pair.Key); - if (property != null) - { - var value = pair.Value; - if (value != null) - { - if (property.PropertyType == typeof(DateTime) || property.PropertyType == typeof(DateTime?)) - { - property.SetValue(orchestrationInstanceStatus, value.DateTime); - } - else if (property.PropertyType == typeof(int) || property.PropertyType == typeof(int?)) - { - property.SetValue(orchestrationInstanceStatus, value.Int32Value); - } - else - { - property.SetValue(orchestrationInstanceStatus, value.StringValue); - } - } - } - } - - return orchestrationInstanceStatus; + return ConvertFromAsync(tableEntity, instanceId, cancellationToken); } - async Task ConvertFromAsync(OrchestrationInstanceStatus orchestrationInstanceStatus, string instanceId) + async Task ConvertFromAsync(OrchestrationInstanceStatus orchestrationInstanceStatus, string instanceId, CancellationToken cancellationToken) { var orchestrationState = new OrchestrationState(); if (!Enum.TryParse(orchestrationInstanceStatus.RuntimeStatus, out orchestrationState.OrchestrationStatus)) @@ -537,7 +465,7 @@ async Task ConvertFromAsync(OrchestrationInstanceStatus orch { if (MessageManager.TryGetLargeMessageReference(orchestrationState.Input, out Uri blobUrl)) { - string json = await this.messageManager.DownloadAndDecompressAsBytesAsync(blobUrl); + string json = await this.messageManager.DownloadAndDecompressAsBytesAsync(blobUrl, cancellationToken); // Depending on which blob this is, we interpret it differently. if (blobUrl.AbsolutePath.EndsWith("ExecutionStarted.json.gz")) @@ -565,175 +493,126 @@ async Task ConvertFromAsync(OrchestrationInstanceStatus orch } } - orchestrationState.Output = await this.messageManager.FetchLargeMessageIfNecessary(orchestrationState.Output); + orchestrationState.Output = await this.messageManager.FetchLargeMessageIfNecessary(orchestrationState.Output, cancellationToken); } return orchestrationState; } /// - public override async Task> GetStateAsync(IEnumerable instanceIds) + public override async IAsyncEnumerable GetStateAsync(IEnumerable instanceIds, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - if (instanceIds == null || !instanceIds.Any()) + if (instanceIds == null) { - return Array.Empty(); + yield break; } - // In theory this could exceed MaxStorageOperationConcurrency, but the hard maximum of parallel requests is tied to control queue - // batch size, which is generally roughly the same value as MaxStorageOperationConcurrency. In almost every case, we would expect this - // to only be a small handful of parallel requests, so keeping the code simple until the storage refactor adds global throttling. - var instanceQueries = instanceIds.Select(instance => this.GetStateAsync(instance, allExecutions: true, fetchInput: false)); - IEnumerable> instanceQueriesResults = await Task.WhenAll(instanceQueries); - return instanceQueriesResults.SelectMany(result => result).Where(orchestrationState => orchestrationState != null).ToList(); + IEnumerable> instanceQueries = instanceIds.Select(instance => this.GetStateAsync(instance, allExecutions: true, fetchInput: false, cancellationToken).SingleAsync().AsTask()); + foreach (OrchestrationState state in await Task.WhenAll(instanceQueries)) + { + if (state != null) + { + yield return state; + } + } } /// - public override Task> GetStateAsync(CancellationToken cancellationToken = default(CancellationToken)) + public override IAsyncEnumerable GetStateAsync(CancellationToken cancellationToken = default) { - TableQuery query = new TableQuery(). - Where(TableQuery.GenerateFilterCondition(RowKeyProperty, QueryComparisons.Equal, string.Empty)); - return this.QueryStateAsync(query, cancellationToken); + return this.QueryStateAsync($"{nameof(ITableEntity.RowKey)} eq ''", cancellationToken: cancellationToken); } - public override Task> GetStateAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, CancellationToken cancellationToken = default(CancellationToken)) + public override AsyncPageable GetStateAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, CancellationToken cancellationToken = default) { - return this.QueryStateAsync(OrchestrationInstanceStatusQueryCondition.Parse(createdTimeFrom, createdTimeTo, runtimeStatus) - .ToTableQuery(), cancellationToken); + ODataCondition odata = OrchestrationInstanceStatusQueryCondition.Parse(createdTimeFrom, createdTimeTo, runtimeStatus).ToOData(); + return this.QueryStateAsync(odata.Filter, odata.Select, cancellationToken); } - public override Task GetStateAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, int top, string continuationToken, CancellationToken cancellationToken = default(CancellationToken)) + public override AsyncPageable GetStateAsync(OrchestrationInstanceStatusQueryCondition condition, CancellationToken cancellationToken = default) { - return this.QueryStateAsync( - OrchestrationInstanceStatusQueryCondition.Parse(createdTimeFrom, createdTimeTo, runtimeStatus) - .ToTableQuery(), - top, - continuationToken, - cancellationToken); + ODataCondition odata = condition.ToOData(); + return this.QueryStateAsync(odata.Filter, odata.Select, cancellationToken); } - public override Task GetStateAsync(OrchestrationInstanceStatusQueryCondition condition, int top, string continuationToken, CancellationToken cancellationToken = default(CancellationToken)) + AsyncPageable QueryStateAsync(string filter = null, IEnumerable select = null, CancellationToken cancellationToken = default) { - return this.QueryStateAsync( - condition.ToTableQuery(), - top, - continuationToken, - cancellationToken); - } - - async Task QueryStateAsync(TableQuery query, int top, string continuationToken, CancellationToken cancellationToken) - { - var orchestrationStates = new List(top); - query.Take(top); - - var tableEntitiesResponseInfo = await this.InstancesTable.ExecuteQuerySegmentAsync(query, cancellationToken, continuationToken); - - IEnumerable result = await Task.WhenAll(tableEntitiesResponseInfo.ReturnedEntities.Select( status => this.ConvertFromAsync(status, KeySanitation.UnescapePartitionKey(status.PartitionKey)))); - orchestrationStates.AddRange(result); - - var queryResult = new DurableStatusQueryResult() - { - OrchestrationState = orchestrationStates, - ContinuationToken = tableEntitiesResponseInfo.ContinuationToken, - }; - - return queryResult; - } - - async Task> QueryStateAsync(TableQuery query, CancellationToken cancellationToken) - { - var orchestrationStates = new List(100); - - var tableEntitiesResponseInfo = await this.InstancesTable.ExecuteQueryAsync(query, cancellationToken); - - IEnumerable result = await Task.WhenAll(tableEntitiesResponseInfo.ReturnedEntities.Select( - status => this.ConvertFromAsync(status, KeySanitation.UnescapePartitionKey(status.PartitionKey)))); - - orchestrationStates.AddRange(result); - - return orchestrationStates; + return new AsyncPageableAsyncProjection( + this.InstancesTable.ExecuteQueryAsync(filter, select: select, cancellationToken: cancellationToken), + (s, t) => new ValueTask(this.ConvertFromAsync(s, KeySanitation.UnescapePartitionKey(s.PartitionKey), t))); } async Task DeleteHistoryAsync( DateTime createdTimeFrom, DateTime? createdTimeTo, - IEnumerable runtimeStatus) + IEnumerable runtimeStatus, + CancellationToken cancellationToken) { - var filter = OrchestrationInstanceStatusQueryCondition.Parse( + var condition = OrchestrationInstanceStatusQueryCondition.Parse( createdTimeFrom, createdTimeTo, runtimeStatus); - filter.FetchInput = false; - filter.FetchOutput = false; + condition.FetchInput = false; + condition.FetchOutput = false; - TableQuery query = filter.ToTableQuery(); + ODataCondition odata = condition.ToOData(); // Limit to batches of 100 to avoid excessive memory usage and table storage scanning - query.TakeCount = 100; - int storageRequests = 0; int instancesDeleted = 0; int rowsDeleted = 0; - while (true) + var options = new ParallelOptions { MaxDegreeOfParallelism = this.settings.MaxStorageOperationConcurrency }; + AsyncPageable entitiesPageable = this.InstancesTable.ExecuteQueryAsync(odata.Filter, select: odata.Select, cancellationToken: cancellationToken); + await foreach (Page page in entitiesPageable.AsPages(pageSizeHint: 100)) { - TableEntitiesResponseInfo tableEntitiesResponseInfo = - await this.InstancesTable.ExecuteQueryAsync(query); - IList results = tableEntitiesResponseInfo.ReturnedEntities; - if (results.Count == 0) + // The underlying client throttles + await Task.WhenAll(page.Values.Select(async instance => { - break; - } - - // Delete instances in parallel - await results.ParallelForEachAsync( - this.settings.MaxStorageOperationConcurrency, - async instance => - { - PurgeHistoryResult statisticsFromDeletion = await this.DeleteAllDataForOrchestrationInstance(instance); - Interlocked.Add(ref instancesDeleted, statisticsFromDeletion.InstancesDeleted); - Interlocked.Add(ref storageRequests, statisticsFromDeletion.RowsDeleted); - Interlocked.Add(ref rowsDeleted, statisticsFromDeletion.RowsDeleted); - }); + PurgeHistoryResult statisticsFromDeletion = await this.DeleteAllDataForOrchestrationInstance(instance, cancellationToken); + Interlocked.Add(ref instancesDeleted, statisticsFromDeletion.InstancesDeleted); + Interlocked.Add(ref storageRequests, statisticsFromDeletion.RowsDeleted); + Interlocked.Add(ref rowsDeleted, statisticsFromDeletion.RowsDeleted); + })); } return new PurgeHistoryResult(storageRequests, instancesDeleted, rowsDeleted); } - async Task DeleteAllDataForOrchestrationInstance(OrchestrationInstanceStatus orchestrationInstanceStatus) + async Task DeleteAllDataForOrchestrationInstance(OrchestrationInstanceStatus orchestrationInstanceStatus, CancellationToken cancellationToken) { int storageRequests = 0; int rowsDeleted = 0; string sanitizedInstanceId = KeySanitation.UnescapePartitionKey(orchestrationInstanceStatus.PartitionKey); - var historyEntitiesResponseInfo = await this.GetHistoryEntitiesResponseInfoAsync( - instanceId: sanitizedInstanceId, - expectedExecutionId: null, - projectionColumns: new[] { RowKeyProperty }); - storageRequests += historyEntitiesResponseInfo.RequestCount; - - IList historyEntities = historyEntitiesResponseInfo.ReturnedEntities; + TableQueryResults results = await this + .GetHistoryEntitiesResponseInfoAsync( + instanceId: sanitizedInstanceId, + expectedExecutionId: null, + projectionColumns: new[] { RowKeyProperty, PartitionKeyProperty, TimestampProperty }, + cancellationToken) + .GetResultsAsync(cancellationToken: cancellationToken); - var tasks = new List(); - tasks.Add(Task.Run(async () => - { - int storageOperations = await this.messageManager.DeleteLargeMessageBlobs(sanitizedInstanceId); - Interlocked.Add(ref storageRequests, storageOperations); - })); + storageRequests += results.RequestCount; - tasks.Add(Task.Run(async () => - { - var deletedEntitiesResponseInfo = await this.HistoryTable.DeleteBatchAsync(historyEntities); - Interlocked.Add(ref rowsDeleted, deletedEntitiesResponseInfo.TableResults.Count); - Interlocked.Add(ref storageRequests, deletedEntitiesResponseInfo.RequestCount); - })); + IReadOnlyList historyEntities = results.Entities; - tasks.Add(this.InstancesTable.DeleteAsync(new DynamicTableEntity + var tasks = new List { - PartitionKey = orchestrationInstanceStatus.PartitionKey, - RowKey = string.Empty, - ETag = "*" - })); + Task.Run(async () => + { + int storageOperations = await this.messageManager.DeleteLargeMessageBlobs(sanitizedInstanceId, cancellationToken); + Interlocked.Add(ref storageRequests, storageOperations); + }), + Task.Run(async () => + { + var deletedEntitiesResponseInfo = await this.HistoryTable.DeleteBatchAsync(historyEntities, cancellationToken); + Interlocked.Add(ref rowsDeleted, deletedEntitiesResponseInfo.Responses.Count); + Interlocked.Add(ref storageRequests, deletedEntitiesResponseInfo.RequestCount); + }), + this.InstancesTable.DeleteAsync(new TableEntity(orchestrationInstanceStatus.PartitionKey, string.Empty), ETag.All, cancellationToken: cancellationToken) + }; await Task.WhenAll(tasks); @@ -744,29 +623,26 @@ async Task DeleteAllDataForOrchestrationInstance(Orchestrati } /// - public override Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType) + public override Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType, CancellationToken cancellationToken = default) { throw new NotSupportedException(); } /// - public override async Task PurgeInstanceHistoryAsync(string instanceId) + public override async Task PurgeInstanceHistoryAsync(string instanceId, CancellationToken cancellationToken = default) { string sanitizedInstanceId = KeySanitation.EscapePartitionKey(instanceId); - TableQuery query = new TableQuery().Where( - TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition(PartitionKeyProperty, QueryComparisons.Equal, sanitizedInstanceId), - TableOperators.And, - TableQuery.GenerateFilterCondition(RowKeyProperty, QueryComparisons.Equal, string.Empty))); - - var tableEntitiesResponseInfo = await this.InstancesTable.ExecuteQueryAsync(query); + string filter = $"{PartitionKeyProperty} eq '{sanitizedInstanceId}' and {RowKeyProperty} eq ''"; + var results = await this.InstancesTable + .ExecuteQueryAsync(filter, cancellationToken: cancellationToken) + .GetResultsAsync(cancellationToken: cancellationToken); - OrchestrationInstanceStatus orchestrationInstanceStatus = tableEntitiesResponseInfo.ReturnedEntities.FirstOrDefault(); + OrchestrationInstanceStatus orchestrationInstanceStatus = results.Entities.FirstOrDefault(); if (orchestrationInstanceStatus != null) { - PurgeHistoryResult result = await this.DeleteAllDataForOrchestrationInstance(orchestrationInstanceStatus); + PurgeHistoryResult result = await this.DeleteAllDataForOrchestrationInstance(orchestrationInstanceStatus, cancellationToken); this.settings.Logger.PurgeInstanceHistory( this.storageAccountName, @@ -777,7 +653,7 @@ public override async Task PurgeInstanceHistoryAsync(string string.Empty, result.StorageRequests, result.InstancesDeleted, - tableEntitiesResponseInfo.ElapsedMilliseconds); + results.ElapsedMilliseconds); return result; } @@ -789,7 +665,8 @@ public override async Task PurgeInstanceHistoryAsync(string public override async Task PurgeInstanceHistoryAsync( DateTime createdTimeFrom, DateTime? createdTimeTo, - IEnumerable runtimeStatus) + IEnumerable runtimeStatus, + CancellationToken cancellationToken = default) { Stopwatch stopwatch = Stopwatch.StartNew(); List runtimeStatusList = runtimeStatus?.Where( @@ -798,7 +675,7 @@ public override async Task PurgeInstanceHistoryAsync(string status == OrchestrationStatus.Canceled || status == OrchestrationStatus.Failed).ToList(); - PurgeHistoryResult result = await this.DeleteHistoryAsync(createdTimeFrom, createdTimeTo, runtimeStatusList); + PurgeHistoryResult result = await this.DeleteHistoryAsync(createdTimeFrom, createdTimeTo, runtimeStatusList, cancellationToken); this.settings.Logger.PurgeInstanceHistory( this.storageAccountName, @@ -819,31 +696,28 @@ public override async Task PurgeInstanceHistoryAsync(string /// public override async Task SetNewExecutionAsync( ExecutionStartedEvent executionStartedEvent, - string eTag, - string inputPayloadOverride) + ETag? eTag, + string inputPayloadOverride, + CancellationToken cancellationToken = default) { string sanitizedInstanceId = KeySanitation.EscapePartitionKey(executionStartedEvent.OrchestrationInstance.InstanceId); - DynamicTableEntity entity = new DynamicTableEntity(sanitizedInstanceId, "") + TableEntity entity = new TableEntity(sanitizedInstanceId, "") { - ETag = eTag, - Properties = - { - ["Input"] = new EntityProperty(inputPayloadOverride ?? executionStartedEvent.Input), - ["CreatedTime"] = new EntityProperty(executionStartedEvent.Timestamp), - ["Name"] = new EntityProperty(executionStartedEvent.Name), - ["Version"] = new EntityProperty(executionStartedEvent.Version), - ["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.Pending.ToString()), - ["LastUpdatedTime"] = new EntityProperty(DateTime.UtcNow), - ["TaskHubName"] = new EntityProperty(this.settings.TaskHubName), - ["ScheduledStartTime"] = new EntityProperty(executionStartedEvent.ScheduledStartTime), - ["ExecutionId"] = new EntityProperty(executionStartedEvent.OrchestrationInstance.ExecutionId), - ["Generation"] = new EntityProperty(executionStartedEvent.Generation), - } + ["Input"] = inputPayloadOverride ?? executionStartedEvent.Input, + ["CreatedTime"] = executionStartedEvent.Timestamp, + ["Name"] = executionStartedEvent.Name, + ["Version"] = executionStartedEvent.Version, + ["RuntimeStatus"] = OrchestrationStatus.Pending.ToString("G"), + ["LastUpdatedTime"] = DateTime.UtcNow, + ["TaskHubName"] = this.settings.TaskHubName, + ["ScheduledStartTime"] = executionStartedEvent.ScheduledStartTime, + ["ExecutionId"] = executionStartedEvent.OrchestrationInstance.ExecutionId, + ["Generation"] = executionStartedEvent.Generation, }; // It is possible that the queue message was small enough to be written directly to a queue message, // not a blob, but is too large to be written to a table property. - await this.CompressLargeMessageAsync(entity); + await this.CompressLargeMessageAsync(entity, cancellationToken); Stopwatch stopwatch = Stopwatch.StartNew(); try @@ -851,12 +725,12 @@ public override async Task PurgeInstanceHistoryAsync(string if (eTag == null) { // This is the case for creating a new instance. - await this.InstancesTable.InsertAsync(entity); + await this.InstancesTable.InsertAsync(entity, cancellationToken); } else { // This is the case for overwriting an existing instance. - await this.InstancesTable.ReplaceAsync(entity); + await this.InstancesTable.ReplaceAsync(entity, eTag.GetValueOrDefault(), cancellationToken); } } catch (DurableTaskStorageException e) when ( @@ -883,21 +757,17 @@ public override async Task PurgeInstanceHistoryAsync(string } /// - public override async Task UpdateStatusForRewindAsync(string instanceId) + public override async Task UpdateStatusForRewindAsync(string instanceId, CancellationToken cancellationToken = default) { string sanitizedInstanceId = KeySanitation.EscapePartitionKey(instanceId); - DynamicTableEntity entity = new DynamicTableEntity(sanitizedInstanceId, "") + TableEntity entity = new TableEntity(sanitizedInstanceId, "") { - ETag = "*", - Properties = - { - ["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.Pending.ToString()), - ["LastUpdatedTime"] = new EntityProperty(DateTime.UtcNow), - } + ["RuntimeStatus"] = OrchestrationStatus.Pending.ToString("G"), + ["LastUpdatedTime"] = DateTime.UtcNow, }; Stopwatch stopwatch = Stopwatch.StartNew(); - await this.InstancesTable.MergeAsync(entity); + await this.InstancesTable.MergeAsync(entity, ETag.All, cancellationToken); // We don't have enough information to get the episode number. // It's also not important to have for this particular trace. @@ -915,20 +785,22 @@ public override async Task UpdateStatusForRewindAsync(string instanceId) /// - public override Task StartAsync() + public override Task StartAsync(CancellationToken cancellationToken = default) { ServicePointManager.FindServicePoint(this.HistoryTable.Uri).UseNagleAlgorithm = false; ServicePointManager.FindServicePoint(this.InstancesTable.Uri).UseNagleAlgorithm = false; - return Utils.CompletedTask; + + return Task.CompletedTask; } /// - public override async Task UpdateStateAsync( + public override async Task UpdateStateAsync( OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, - string eTagValue) + ETag? eTagValue, + CancellationToken cancellationToken = default) { int estimatedBytes = 0; IList newEvents = newRuntimeState.NewEvents; @@ -937,21 +809,18 @@ public override Task StartAsync() int episodeNumber = Utils.GetEpisodeNumber(newRuntimeState); var newEventListBuffer = new StringBuilder(4000); - var historyEventBatch = new TableBatchOperation(); + var historyEventBatch = new List(); OrchestrationStatus runtimeStatus = OrchestrationStatus.Running; string sanitizedInstanceId = KeySanitation.EscapePartitionKey(instanceId); - var instanceEntity = new DynamicTableEntity(sanitizedInstanceId, string.Empty) + var instanceEntity = new TableEntity(sanitizedInstanceId, string.Empty) { - Properties = - { - // TODO: Translating null to "null" is a temporary workaround. We should prioritize - // https://github.com/Azure/durabletask/issues/477 so that this is no longer necessary. - ["CustomStatus"] = new EntityProperty(newRuntimeState.Status ?? "null"), - ["ExecutionId"] = new EntityProperty(executionId), - ["LastUpdatedTime"] = new EntityProperty(newEvents.Last().Timestamp), - } + // TODO: Translating null to "null" is a temporary workaround. We should prioritize + // https://github.com/Azure/durabletask/issues/477 so that this is no longer necessary. + ["CustomStatus"] = newRuntimeState.Status ?? "null", + ["ExecutionId"] = executionId, + ["LastUpdatedTime"] = newEvents.Last().Timestamp, }; for (int i = 0; i < newEvents.Count; i++) @@ -959,7 +828,7 @@ public override Task StartAsync() bool isFinalEvent = i == newEvents.Count - 1; HistoryEvent historyEvent = newEvents[i]; - var historyEntity = this.tableEntityConverter.ConvertToTableEntity(historyEvent); + var historyEntity = TableEntityConverter.Serialize(historyEvent); historyEntity.PartitionKey = sanitizedInstanceId; newEventListBuffer.Append(historyEvent.EventType.ToString()).Append(','); @@ -967,12 +836,12 @@ public override Task StartAsync() // The row key is the sequence number, which represents the chronological ordinal of the event. long sequenceNumber = i + (allEvents.Count - newEvents.Count); historyEntity.RowKey = sequenceNumber.ToString("X16"); - historyEntity.Properties["ExecutionId"] = new EntityProperty(executionId); + historyEntity["ExecutionId"] = executionId; - await this.CompressLargeMessageAsync(historyEntity); + await this.CompressLargeMessageAsync(historyEntity, cancellationToken); // Replacement can happen if the orchestration episode gets replayed due to a commit failure in one of the steps below. - historyEventBatch.InsertOrReplace(historyEntity); + historyEventBatch.Add(new TableTransactionAction(TableTransactionActionType.UpsertReplace, historyEntity)); // Keep track of the byte count to ensure we don't hit the 4 MB per-batch maximum estimatedBytes += GetEstimatedByteCount(historyEntity); @@ -983,13 +852,13 @@ public override Task StartAsync() case EventType.ExecutionStarted: runtimeStatus = OrchestrationStatus.Running; ExecutionStartedEvent executionStartedEvent = (ExecutionStartedEvent)historyEvent; - instanceEntity.Properties["Name"] = new EntityProperty(executionStartedEvent.Name); - instanceEntity.Properties["Version"] = new EntityProperty(executionStartedEvent.Version); - instanceEntity.Properties["CreatedTime"] = new EntityProperty(executionStartedEvent.Timestamp); - instanceEntity.Properties["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.Running.ToString()); + instanceEntity["Name"] = executionStartedEvent.Name; + instanceEntity["Version"] = executionStartedEvent.Version; + instanceEntity["CreatedTime"] = executionStartedEvent.Timestamp; + instanceEntity["RuntimeStatus"] = OrchestrationStatus.Running.ToString(); if (executionStartedEvent.ScheduledStartTime.HasValue) { - instanceEntity.Properties["ScheduledStartTime"] = new EntityProperty(executionStartedEvent.ScheduledStartTime); + instanceEntity["ScheduledStartTime"] = executionStartedEvent.ScheduledStartTime; } this.SetInstancesTablePropertyFromHistoryProperty( @@ -1002,8 +871,8 @@ public override Task StartAsync() case EventType.ExecutionCompleted: ExecutionCompletedEvent executionCompleted = (ExecutionCompletedEvent)historyEvent; runtimeStatus = executionCompleted.OrchestrationStatus; - instanceEntity.Properties["RuntimeStatus"] = new EntityProperty(executionCompleted.OrchestrationStatus.ToString()); - instanceEntity.Properties["CompletedTime"] = new EntityProperty(DateTime.UtcNow); + instanceEntity["RuntimeStatus"] = executionCompleted.OrchestrationStatus.ToString(); + instanceEntity["CompletedTime"] = DateTime.UtcNow; this.SetInstancesTablePropertyFromHistoryProperty( historyEntity, instanceEntity, @@ -1014,8 +883,8 @@ public override Task StartAsync() case EventType.ExecutionTerminated: runtimeStatus = OrchestrationStatus.Terminated; ExecutionTerminatedEvent executionTerminatedEvent = (ExecutionTerminatedEvent)historyEvent; - instanceEntity.Properties["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.Terminated.ToString()); - instanceEntity.Properties["CompletedTime"] = new EntityProperty(DateTime.UtcNow); + instanceEntity["RuntimeStatus"] = OrchestrationStatus.Terminated.ToString(); + instanceEntity["CompletedTime"] = DateTime.UtcNow; this.SetInstancesTablePropertyFromHistoryProperty( historyEntity, instanceEntity, @@ -1026,7 +895,7 @@ public override Task StartAsync() case EventType.ExecutionSuspended: runtimeStatus = OrchestrationStatus.Suspended; ExecutionSuspendedEvent executionSuspendedEvent = (ExecutionSuspendedEvent)historyEvent; - instanceEntity.Properties["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.Suspended.ToString()); + instanceEntity["RuntimeStatus"] = OrchestrationStatus.Suspended.ToString(); this.SetInstancesTablePropertyFromHistoryProperty( historyEntity, instanceEntity, @@ -1037,7 +906,7 @@ public override Task StartAsync() case EventType.ExecutionResumed: runtimeStatus = OrchestrationStatus.Running; ExecutionResumedEvent executionResumedEvent = (ExecutionResumedEvent)historyEvent; - instanceEntity.Properties["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.Running.ToString()); + instanceEntity["RuntimeStatus"] = OrchestrationStatus.Running.ToString(); this.SetInstancesTablePropertyFromHistoryProperty( historyEntity, instanceEntity, @@ -1048,7 +917,7 @@ public override Task StartAsync() case EventType.ContinueAsNew: runtimeStatus = OrchestrationStatus.ContinuedAsNew; ExecutionCompletedEvent executionCompletedEvent = (ExecutionCompletedEvent)historyEvent; - instanceEntity.Properties["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.ContinuedAsNew.ToString()); + instanceEntity["RuntimeStatus"] = OrchestrationStatus.ContinuedAsNew.ToString(); this.SetInstancesTablePropertyFromHistoryProperty( historyEntity, instanceEntity, @@ -1071,7 +940,8 @@ public override Task StartAsync() episodeNumber, estimatedBytes, eTagValue, - isFinalBatch: isFinalEvent); + isFinalBatch: isFinalEvent, + cancellationToken: cancellationToken); // Reset local state for the next batch newEventListBuffer.Clear(); @@ -1093,7 +963,8 @@ public override Task StartAsync() episodeNumber, estimatedBytes, eTagValue, - isFinalBatch: true); + isFinalBatch: true, + cancellationToken: cancellationToken); } Stopwatch orchestrationInstanceUpdateStopwatch = Stopwatch.StartNew(); @@ -1111,7 +982,7 @@ public override Task StartAsync() return eTagValue; } - static int GetEstimatedByteCount(DynamicTableEntity entity) + static int GetEstimatedByteCount(TableEntity entity) { // Assume at least 1 KB of data per entity to account for static-length properties int estimatedByteCount = 1024; @@ -1119,35 +990,32 @@ static int GetEstimatedByteCount(DynamicTableEntity entity) // Count the bytes for variable-length properties, which are assumed to always be strings foreach (string propertyName in VariableSizeEntityProperties) { - EntityProperty property; - if (entity.Properties.TryGetValue(propertyName, out property) && !string.IsNullOrEmpty(property.StringValue)) + if (entity.TryGetValue(propertyName, out object property) && property is string stringProperty && stringProperty != "") { - estimatedByteCount += Encoding.Unicode.GetByteCount(property.StringValue); + estimatedByteCount += Encoding.Unicode.GetByteCount(stringProperty); } } return estimatedByteCount; } - Type GetTypeForTableEntity(DynamicTableEntity tableEntity) + Type GetTypeForTableEntity(TableEntity tableEntity) { string propertyName = nameof(HistoryEvent.EventType); - EntityProperty eventTypeProperty; - if (!tableEntity.Properties.TryGetValue(propertyName, out eventTypeProperty)) + if (!tableEntity.TryGetValue(propertyName, out object eventTypeProperty)) { - throw new ArgumentException($"The DynamicTableEntity did not contain a '{propertyName}' property."); + throw new ArgumentException($"The TableEntity did not contain a '{propertyName}' property."); } - if (eventTypeProperty.PropertyType != EdmType.String) + if (eventTypeProperty is not string stringProperty) { - throw new ArgumentException($"The DynamicTableEntity's {propertyName} property type must a String."); + throw new ArgumentException($"The TableEntity's {propertyName} property type must a String."); } - EventType eventType; - if (!Enum.TryParse(eventTypeProperty.StringValue, out eventType)) + if (!Enum.TryParse(stringProperty, out EventType eventType)) { - throw new ArgumentException($"{eventTypeProperty.StringValue} is not a valid EventType value."); + throw new ArgumentException($"{stringProperty} is not a valid EventType value."); } return this.eventTypeMap[eventType]; @@ -1156,60 +1024,59 @@ Type GetTypeForTableEntity(DynamicTableEntity tableEntity) // Assigns the target table entity property. Any large message for type 'Input, or 'Output' would have been compressed earlier as part of the 'entity' object, // so, we only need to assign the 'entity' object's blobName to the target table entity blob name property. void SetInstancesTablePropertyFromHistoryProperty( - DynamicTableEntity historyEntity, - DynamicTableEntity instanceEntity, + TableEntity TableEntity, + TableEntity instanceEntity, string historyPropertyName, string instancePropertyName, string data) { string blobPropertyName = GetBlobPropertyName(historyPropertyName); - if (historyEntity.Properties.TryGetValue(blobPropertyName, out EntityProperty blobProperty)) + if (TableEntity.TryGetValue(blobPropertyName, out object blobProperty) && blobProperty is string blobName) { // This is a large message - string blobName = blobProperty.StringValue; string blobUrl = this.messageManager.GetBlobUrl(blobName); - instanceEntity.Properties[instancePropertyName] = new EntityProperty(blobUrl); + instanceEntity[instancePropertyName] = blobUrl; } else { // This is a normal-sized message and can be stored inline - instanceEntity.Properties[instancePropertyName] = new EntityProperty(data); + instanceEntity[instancePropertyName] = data; } } - async Task CompressLargeMessageAsync(DynamicTableEntity entity) + async Task CompressLargeMessageAsync(TableEntity entity, CancellationToken cancellationToken) { foreach (string propertyName in VariableSizeEntityProperties) { - if (entity.Properties.TryGetValue(propertyName, out EntityProperty property) && - this.ExceedsMaxTablePropertySize(property.StringValue)) + if (entity.TryGetValue(propertyName, out object property) && + property is string stringProperty && + this.ExceedsMaxTablePropertySize(stringProperty)) { // Upload the large property as a blob in Blob Storage since it won't fit in table storage. string blobName = GetBlobName(entity, propertyName); - byte[] messageBytes = Encoding.UTF8.GetBytes(entity.Properties[propertyName].StringValue); - await this.messageManager.CompressAndUploadAsBytesAsync(messageBytes, blobName); + byte[] messageBytes = Encoding.UTF8.GetBytes(stringProperty); + await this.messageManager.CompressAndUploadAsBytesAsync(messageBytes, blobName, cancellationToken); // Clear out the original property value and create a new "*BlobName"-suffixed property. // The runtime will look for the new "*BlobName"-suffixed column to know if a property is stored in a blob. string blobPropertyName = GetBlobPropertyName(propertyName); - entity.Properties.Add(blobPropertyName, new EntityProperty(blobName)); - entity.Properties[propertyName].StringValue = string.Empty; + entity.Add(blobPropertyName, blobName); + entity[propertyName] = string.Empty; } } } - async Task DecompressLargeEntityProperties(DynamicTableEntity entity) + async Task DecompressLargeEntityProperties(TableEntity entity, CancellationToken cancellationToken) { // Check for entity properties stored in blob storage foreach (string propertyName in VariableSizeEntityProperties) { string blobPropertyName = GetBlobPropertyName(propertyName); - if (entity.Properties.TryGetValue(blobPropertyName, out EntityProperty property)) + if (entity.TryGetValue(blobPropertyName, out object property) && property is string blobName) { - string blobName = property.StringValue; - string decompressedMessage = await this.messageManager.DownloadAndDecompressAsBytesAsync(blobName); - entity.Properties[propertyName] = new EntityProperty(decompressedMessage); - entity.Properties.Remove(blobPropertyName); + string decompressedMessage = await this.messageManager.DownloadAndDecompressAsBytesAsync(blobName, cancellationToken); + entity[propertyName] = decompressedMessage; + entity.Remove(blobPropertyName); } } } @@ -1220,15 +1087,15 @@ static string GetBlobPropertyName(string originalPropertyName) return originalPropertyName + "BlobName"; } - static string GetBlobName(DynamicTableEntity entity, string property) + static string GetBlobName(TableEntity entity, string property) { string sanitizedInstanceId = entity.PartitionKey; string sequenceNumber = entity.RowKey; string eventType; - if (entity.Properties.ContainsKey("EventType")) + if (entity.TryGetValue("EventType", out object obj) && obj is string value) { - eventType = entity.Properties["EventType"].StringValue; + eventType = value; } else if (property == "Input") { @@ -1241,53 +1108,48 @@ static string GetBlobName(DynamicTableEntity entity, string property) throw new InvalidOperationException($"Could not compute the blob name for property {property}"); } - string blobName = $"{sanitizedInstanceId}/history-{sequenceNumber}-{eventType}-{property}.json.gz"; - - return blobName; + return $"{sanitizedInstanceId}/history-{sequenceNumber}-{eventType}-{property}.json.gz"; } - async Task UploadHistoryBatch( + async Task UploadHistoryBatch( string instanceId, string sanitizedInstanceId, string executionId, - TableBatchOperation historyEventBatch, + IList historyEventBatch, StringBuilder historyEventNamesBuffer, int numberOfTotalEvents, int episodeNumber, int estimatedBatchSizeInBytes, - string eTagValue, - bool isFinalBatch) + ETag? eTagValue, + bool isFinalBatch, + CancellationToken cancellationToken) { // Adding / updating sentinel entity - DynamicTableEntity sentinelEntity = new DynamicTableEntity(sanitizedInstanceId, SentinelRowKey) + TableEntity sentinelEntity = new TableEntity(sanitizedInstanceId, SentinelRowKey) { - Properties = - { - ["ExecutionId"] = new EntityProperty(executionId), - [IsCheckpointCompleteProperty] = new EntityProperty(isFinalBatch), - } + ["ExecutionId"] = executionId, + [IsCheckpointCompleteProperty] = isFinalBatch, }; if (isFinalBatch) { - sentinelEntity.Properties[CheckpointCompletedTimestampProperty] = new EntityProperty(DateTime.UtcNow); + sentinelEntity[CheckpointCompletedTimestampProperty] = DateTime.UtcNow; } - if (!string.IsNullOrEmpty(eTagValue)) + if (eTagValue != null) { - sentinelEntity.ETag = eTagValue; - historyEventBatch.Merge(sentinelEntity); + historyEventBatch.Add(new TableTransactionAction(TableTransactionActionType.UpdateMerge, sentinelEntity, eTagValue.GetValueOrDefault())); } else { - historyEventBatch.Insert(sentinelEntity); + historyEventBatch.Add(new TableTransactionAction(TableTransactionActionType.Add, sentinelEntity)); } - TableResultResponseInfo resultInfo; + TableTransactionResults resultInfo; Stopwatch stopwatch = Stopwatch.StartNew(); try { - resultInfo = await this.HistoryTable.ExecuteBatchAsync(historyEventBatch, "InsertOrMerge History"); + resultInfo = await this.HistoryTable.ExecuteBatchAsync(historyEventBatch, cancellationToken); } catch (DurableTaskStorageException ex) { @@ -1302,20 +1164,19 @@ static string GetBlobName(DynamicTableEntity entity, string property) numberOfTotalEvents, historyEventNamesBuffer.ToString(0, historyEventNamesBuffer.Length - 1), // remove trailing comma stopwatch.ElapsedMilliseconds, - eTagValue); + eTagValue?.ToString()); } throw; } - var tableResultList = resultInfo.TableResults; - string newETagValue = null; - for (int i = tableResultList.Count - 1; i >= 0; i--) + IReadOnlyList responses = resultInfo.Responses; + ETag? newETagValue = null; + for (int i = responses.Count - 1; i >= 0; i--) { - DynamicTableEntity resultEntity = (DynamicTableEntity)tableResultList[i].Result; - if (resultEntity.RowKey == SentinelRowKey) + if (historyEventBatch[i].Entity.RowKey == SentinelRowKey) { - newETagValue = resultEntity.ETag; + newETagValue = responses[i].Headers.ETag; break; } } @@ -1331,7 +1192,7 @@ static string GetBlobName(DynamicTableEntity entity, string property) episodeNumber, resultInfo.ElapsedMilliseconds, estimatedBatchSizeInBytes, - string.Concat(eTagValue ?? "(null)", " --> ", newETagValue ?? "(null)"), + string.Concat(eTagValue?.ToString() ?? "(null)", " --> ", newETagValue?.ToString() ?? "(null)"), isFinalBatch); return newETagValue; diff --git a/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs index b9d48052a..ca8d8de2e 100644 --- a/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs @@ -17,6 +17,7 @@ namespace DurableTask.AzureStorage.Tracking using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; + using Azure; using DurableTask.Core; using DurableTask.Core.History; @@ -28,38 +29,41 @@ interface ITrackingStore /// /// Create Tracking Store Resources if they don't already exist /// - Task CreateAsync(); + /// The token to monitor for cancellation requests. The default value is . + Task CreateAsync(CancellationToken cancellationToken = default); /// /// Delete Tracking Store Resources if they already exist /// - Task DeleteAsync(); + /// The token to monitor for cancellation requests. The default value is . + Task DeleteAsync(CancellationToken cancellationToken = default); /// /// Do the Resources for the tracking store already exist /// - Task ExistsAsync(); + /// The token to monitor for cancellation requests. The default value is . + Task ExistsAsync(CancellationToken cancellationToken = default); /// /// Start up the Tracking Store before use /// - Task StartAsync(); + /// The token to monitor for cancellation requests. The default value is . + Task StartAsync(CancellationToken cancellationToken = default); /// /// Get History Events from the Store /// /// InstanceId for /// ExcutionId for the execution that we want this retrieve for. If null the latest execution will be retrieved - /// CancellationToken if abortion is needed - Task GetHistoryEventsAsync(string instanceId, string expectedExecutionId, CancellationToken cancellationToken = default(CancellationToken)); + /// The token to monitor for cancellation requests. The default value is . + Task GetHistoryEventsAsync(string instanceId, string expectedExecutionId, CancellationToken cancellationToken = default); /// /// Queries by InstanceId and locates failure - then calls function to wipe ExecutionIds /// /// InstanceId for orchestration - /// List of failed orchestrators to send to message queue - no failed sub-orchestrators - /// CancellationToken if abortion is needed - Task> RewindHistoryAsync(string instanceId, IList failedLeaves, CancellationToken cancellationToken); + /// The token to monitor for cancellation requests. The default value is . + IAsyncEnumerable RewindHistoryAsync(string instanceId, CancellationToken cancellationToken = default); /// /// Update State in the Tracking store for a particular orchestration instance and execution base on the new runtime state @@ -69,7 +73,8 @@ interface ITrackingStore /// InstanceId for the Orchestration Update /// ExecutionId for the Orchestration Update /// The ETag value to use for safe updates - Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, string eTag); + /// The token to monitor for cancellation requests. The default value is . + Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, ETag? eTag, CancellationToken cancellationToken = default); /// /// Get The Orchestration State for the Latest or All Executions @@ -77,7 +82,8 @@ interface ITrackingStore /// Instance Id /// True if states for all executions are to be fetched otherwise only the state for the latest execution of the instance is fetched /// If set, fetch and return the input for the orchestration instance. - Task> GetStateAsync(string instanceId, bool allExecutions, bool fetchInput); + /// The token to monitor for cancellation requests. The default value is . + IAsyncEnumerable GetStateAsync(string instanceId, bool allExecutions, bool fetchInput, CancellationToken cancellationToken = default); /// /// Get The Orchestration State for a particular orchestration instance execution @@ -85,26 +91,30 @@ interface ITrackingStore /// Instance Id /// Execution Id /// If set, fetch and return the input for the orchestration instance. - Task GetStateAsync(string instanceId, string executionId, bool fetchInput); + /// The token to monitor for cancellation requests. The default value is . + Task GetStateAsync(string instanceId, string executionId, bool fetchInput, CancellationToken cancellationToken = default); /// /// Fetches the latest instance status of the specified orchestration instance. /// /// The ID of the orchestration. + /// The token to monitor for cancellation requests. The default value is . /// Returns the instance status or null if none was found. - Task FetchInstanceStatusAsync(string instanceId); + Task FetchInstanceStatusAsync(string instanceId, CancellationToken cancellationToken = default); /// /// Get The Orchestration State for querying all orchestration instances /// + /// The token to monitor for cancellation requests. The default value is . /// - Task> GetStateAsync(CancellationToken cancellationToken = default(CancellationToken)); + IAsyncEnumerable GetStateAsync(CancellationToken cancellationToken = default); /// /// Fetches instances status for multiple orchestration instances. /// /// The list of instances to query for. - Task> GetStateAsync(IEnumerable instanceIds); + /// The token to monitor for cancellation requests. The default value is . + IAsyncEnumerable GetStateAsync(IEnumerable instanceIds, CancellationToken cancellationToken = default); /// /// Get The Orchestration State for querying orchestration instances by the condition @@ -112,31 +122,17 @@ interface ITrackingStore /// CreatedTimeFrom /// CreatedTimeTo /// RuntimeStatus - /// cancellation token + /// The token to monitor for cancellation requests. The default value is . /// - Task> GetStateAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, CancellationToken cancellationToken = default(CancellationToken)); - - /// - /// Get The Orchestration State for querying orchestration instances by the condition - /// - /// CreatedTimeFrom - /// CreatedTimeTo - /// RuntimeStatus - /// Top - /// Continuation token - /// cancellation token - /// - Task GetStateAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, int top, string continuationToken, CancellationToken cancellationToken = default(CancellationToken)); + AsyncPageable GetStateAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, CancellationToken cancellationToken = default); /// /// Get The Orchestration State for querying orchestration instances by the condition /// /// Condition - /// Top - /// ContinuationToken - /// CancellationToken + /// The token to monitor for cancellation requests. The default value is . /// - Task GetStateAsync(OrchestrationInstanceStatusQueryCondition condition, int top, string continuationToken, CancellationToken cancellationToken = default(CancellationToken)); + AsyncPageable GetStateAsync(OrchestrationInstanceStatusQueryCondition condition, CancellationToken cancellationToken = default); /// /// Used to set a state in the tracking store whenever a new execution is initiated from the client @@ -144,28 +140,32 @@ interface ITrackingStore /// The Execution Started Event being queued /// The eTag value to use for optimistic concurrency or null to overwrite any existing execution status. /// An override value to use for the Input column. If not specified, uses . + /// The token to monitor for cancellation requests. The default value is . /// Returns true if the record was created successfully; false otherwise. - Task SetNewExecutionAsync(ExecutionStartedEvent executionStartedEvent, string eTag, string inputPayloadOverride); + Task SetNewExecutionAsync(ExecutionStartedEvent executionStartedEvent, ETag? eTag, string inputPayloadOverride, CancellationToken cancellationToken = default); /// /// Used to update a state in the tracking store to pending whenever a rewind is initiated from the client /// /// The instance being rewound - Task UpdateStatusForRewindAsync(string instanceId); + /// The token to monitor for cancellation requests. The default value is . + Task UpdateStatusForRewindAsync(string instanceId, CancellationToken cancellationToken = default); /// /// Purge The History and state which is older than thresholdDateTimeUtc based on the timestamp type specified by timeRangeFilterType /// /// Timestamp threshold, data older than this will be removed /// timeRangeFilterType governs the type of time stamp that will be used for decision making - Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType); + /// The token to monitor for cancellation requests. The default value is . + Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType, CancellationToken cancellationToken = default); /// /// Purge the history for a concrete instance /// /// Instance ID + /// The token to monitor for cancellation requests. The default value is . /// Class containing number of storage requests sent, along with instances and rows deleted/purged - Task PurgeInstanceHistoryAsync(string instanceId); + Task PurgeInstanceHistoryAsync(string instanceId, CancellationToken cancellationToken = default); /// /// Purge the orchestration history for instances that match the conditions @@ -173,7 +173,8 @@ interface ITrackingStore /// Start creation time for querying instances for purging /// End creation time for querying instances for purging /// List of runtime status for querying instances for purging. Only Completed, Terminated, or Failed will be processed + /// The token to monitor for cancellation requests. The default value is . /// Class containing number of storage requests sent, along with instances and rows deleted/purged - Task PurgeInstanceHistoryAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus); + Task PurgeInstanceHistoryAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, CancellationToken cancellationToken = default); } } diff --git a/src/DurableTask.AzureStorage/Tracking/InstanceStatus.cs b/src/DurableTask.AzureStorage/Tracking/InstanceStatus.cs index e75cd132f..0fcf6198b 100644 --- a/src/DurableTask.AzureStorage/Tracking/InstanceStatus.cs +++ b/src/DurableTask.AzureStorage/Tracking/InstanceStatus.cs @@ -12,6 +12,7 @@ // ---------------------------------------------------------------------------------- using System; +using Azure; using DurableTask.Core; namespace DurableTask.AzureStorage.Tracking @@ -22,14 +23,14 @@ public InstanceStatus(OrchestrationState state) : this(state, null) { } - public InstanceStatus(OrchestrationState state, string eTag) + public InstanceStatus(OrchestrationState state, ETag? eTag) { this.State = state ?? throw new ArgumentNullException(nameof(state)); - this.ETag = eTag ?? "*"; + this.ETag = eTag ?? ETag.All; } public OrchestrationState State { get; } - public string ETag { get; } + public ETag ETag { get; } } } diff --git a/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs index 1e7da45ac..9a41d234e 100644 --- a/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs @@ -16,8 +16,10 @@ namespace DurableTask.AzureStorage.Tracking using System; using System.Collections.Generic; using System.Linq; + using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; + using Azure; using DurableTask.Core; using DurableTask.Core.History; using DurableTask.Core.Tracking; @@ -33,13 +35,13 @@ public InstanceStoreBackedTrackingStore(IOrchestrationServiceInstanceStore insta } /// - public override Task CreateAsync() + public override Task CreateAsync(CancellationToken cancellationToken = default) { return this.instanceStore.InitializeStoreAsync(false); } /// - public override Task DeleteAsync() + public override Task DeleteAsync(CancellationToken cancellationToken = default) { return this.instanceStore.DeleteStoreAsync(); } @@ -54,37 +56,33 @@ public override async Task GetHistoryEventsAsync(string in } var events = await this.instanceStore.GetOrchestrationHistoryEventsAsync(instanceId, expectedExecutionId); - - if (events == null || !events.Any()) - { - return new OrchestrationHistory(EmptyHistoryEventList); - } - else - { - return new OrchestrationHistory(events.Select(x => x.HistoryEvent).ToList()); - } + IList history = events?.Select(x => x.HistoryEvent).ToList(); + return new OrchestrationHistory(history ?? Array.Empty()); } /// - public override async Task FetchInstanceStatusAsync(string instanceId) + public override async Task FetchInstanceStatusAsync(string instanceId, CancellationToken cancellationToken = default) { - OrchestrationState state = await this.GetStateAsync(instanceId, executionId: null); + OrchestrationState state = await this.GetStateAsync(instanceId, executionId: null, cancellationToken: cancellationToken); return state != null ? new InstanceStatus(state) : null; } /// - public override async Task> GetStateAsync(string instanceId, bool allExecutions, bool fetchInput = true) + public override async IAsyncEnumerable GetStateAsync(string instanceId, bool allExecutions, bool fetchInput = true, [EnumeratorCancellation] CancellationToken cancellationToken = default) { IEnumerable states = await instanceStore.GetOrchestrationStateAsync(instanceId, allExecutions); - return states?.Select(s => s.State).ToList() ?? new List(); + foreach (var s in states ?? Array.Empty()) + { + yield return s.State; + } } /// - public override async Task GetStateAsync(string instanceId, string executionId, bool fetchInput = true) + public override async Task GetStateAsync(string instanceId, string executionId, bool fetchInput = true, CancellationToken cancellationToken = default) { if (executionId == null) { - return (await this.GetStateAsync(instanceId, false)).FirstOrDefault(); + return await this.GetStateAsync(instanceId, false, cancellationToken: cancellationToken).FirstOrDefaultAsync(cancellationToken); } else { @@ -93,7 +91,7 @@ public override async Task GetStateAsync(string instanceId, } /// - public override Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType) + public override Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType, CancellationToken cancellationToken = default) { return this.instanceStore.PurgeOrchestrationHistoryEventsAsync(thresholdDateTimeUtc, timeRangeFilterType); } @@ -101,8 +99,9 @@ public override Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, Orchestrat /// public override async Task SetNewExecutionAsync( ExecutionStartedEvent executionStartedEvent, - string eTag /* not used */, - string inputStatusOverride) + ETag? eTag /* not used */, + string inputStatusOverride, + CancellationToken cancellationToken = default) { var orchestrationState = new OrchestrationState() { @@ -130,27 +129,27 @@ public override Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, Orchestrat } /// - public override Task StartAsync() + public override Task StartAsync(CancellationToken cancellationToken = default) { //NOP return Utils.CompletedTask; } /// - public override async Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, string eTag) + public override async Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, ETag? eTag, CancellationToken cancellationToken = default) { //In case there is a runtime state for an older execution/iteration as well that needs to be committed, commit it. //This may be the case if a ContinueAsNew was executed on the orchestration if (newRuntimeState != oldRuntimeState) { - eTag = await UpdateStateAsync(oldRuntimeState, instanceId, oldRuntimeState.OrchestrationInstance.ExecutionId, eTag); + eTag = await UpdateStateAsync(oldRuntimeState, instanceId, oldRuntimeState.OrchestrationInstance.ExecutionId, eTag, cancellationToken); } - return await UpdateStateAsync(newRuntimeState, instanceId, executionId, eTag); + return await UpdateStateAsync(newRuntimeState, instanceId, executionId, eTag, cancellationToken); } /// - private async Task UpdateStateAsync(OrchestrationRuntimeState runtimeState, string instanceId, string executionId, string eTag) + private async Task UpdateStateAsync(OrchestrationRuntimeState runtimeState, string instanceId, string executionId, ETag? eTag, CancellationToken cancellationToken = default) { int oldEventsCount = (runtimeState.Events.Count - runtimeState.NewEvents.Count); await instanceStore.WriteEntitiesAsync(runtimeState.NewEvents.Select((x, i) => diff --git a/src/DurableTask.AzureStorage/Tracking/ODataCondition.cs b/src/DurableTask.AzureStorage/Tracking/ODataCondition.cs new file mode 100644 index 000000000..6732c3a6b --- /dev/null +++ b/src/DurableTask.AzureStorage/Tracking/ODataCondition.cs @@ -0,0 +1,46 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Tracking +{ + using System.Collections.Generic; + + /// + /// Represents an OData condition as specified by a filter and a projection. + /// + public readonly struct ODataCondition + { + /// + /// Gets the optional collection of properties to be projected. + /// + /// An optional enumerable of zero or more property names. + public IEnumerable? Select { get; } + + /// + /// Gets the optional filter expression. + /// + /// An optional string that represents the filter expression. + public string? Filter { get; } + + /// + /// Initializes a new instance of the structure based on the specified values. + /// + /// An optional collection of properties to be projected. + /// An optional filter expression. + public ODataCondition(IEnumerable? select = null, string? filter = null) + { + Select = select; + Filter = filter; + } + } +} diff --git a/src/DurableTask.AzureStorage/Tracking/OrchestrationHistory.cs b/src/DurableTask.AzureStorage/Tracking/OrchestrationHistory.cs index 750406564..436f9641b 100644 --- a/src/DurableTask.AzureStorage/Tracking/OrchestrationHistory.cs +++ b/src/DurableTask.AzureStorage/Tracking/OrchestrationHistory.cs @@ -15,6 +15,7 @@ namespace DurableTask.AzureStorage.Tracking { using System; using System.Collections.Generic; + using Azure; using DurableTask.Core.History; class OrchestrationHistory @@ -29,7 +30,7 @@ public OrchestrationHistory(IList historyEvents, DateTime lastChec { } - public OrchestrationHistory(IList historyEvents, DateTime lastCheckpointTime, string eTag) + public OrchestrationHistory(IList historyEvents, DateTime lastCheckpointTime, ETag? eTag) { this.Events = historyEvents ?? throw new ArgumentNullException(nameof(historyEvents)); this.LastCheckpointTime = lastCheckpointTime; @@ -38,7 +39,7 @@ public OrchestrationHistory(IList historyEvents, DateTime lastChec public IList Events { get; } - public string ETag { get; } + public ETag? ETag { get; } public DateTime LastCheckpointTime { get; } } diff --git a/src/DurableTask.AzureStorage/Tracking/OrchestrationInstanceStatusQueryCondition.cs b/src/DurableTask.AzureStorage/Tracking/OrchestrationInstanceStatusQueryCondition.cs index ae2ff74b9..cc96f2be4 100644 --- a/src/DurableTask.AzureStorage/Tracking/OrchestrationInstanceStatusQueryCondition.cs +++ b/src/DurableTask.AzureStorage/Tracking/OrchestrationInstanceStatusQueryCondition.cs @@ -17,12 +17,11 @@ namespace DurableTask.AzureStorage.Tracking using System.Collections.Generic; using System.Linq; using DurableTask.Core; - using Microsoft.WindowsAzure.Storage.Table; /// /// OrchestrationInstanceStatusQueryBuilder is a builder to create a StorageTable Query /// - public class OrchestrationInstanceStatusQueryCondition + public sealed class OrchestrationInstanceStatusQueryCondition { static readonly string[] ColumnNames = typeof(OrchestrationInstanceStatus).GetProperties() .Select(prop => prop.Name) @@ -69,21 +68,19 @@ public class OrchestrationInstanceStatusQueryCondition public bool FetchOutput { get; set; } = true; /// - /// Get the TableQuery object + /// Get the corresponding OData filter. /// - /// /// - public TableQuery ToTableQuery() - where T : ITableEntity, new() + internal ODataCondition ToOData() { - var query = new TableQuery(); - if (!((this.RuntimeStatus == null || !this.RuntimeStatus.Any()) && - this.CreatedTimeFrom == default(DateTime) && + if (!((this.RuntimeStatus == null || !this.RuntimeStatus.Any()) && + this.CreatedTimeFrom == default(DateTime) && this.CreatedTimeTo == default(DateTime) && this.TaskHubNames == null && this.InstanceIdPrefix == null && this.InstanceId == null)) { + IEnumerable? select = null; if (!this.FetchInput || !this.FetchOutput) { var columns = new HashSet(ColumnNames); @@ -97,53 +94,36 @@ public TableQuery ToTableQuery() columns.Remove(nameof(OrchestrationInstanceStatus.Output)); } - query.Select(columns.ToList()); + select = columns; } - string conditions = this.GetConditions(); - if (!string.IsNullOrEmpty(conditions)) - { - query.Where(conditions); - } + return new ODataCondition(select, this.GetODataFilter()); } - return query; + return default; } - string GetConditions() + string? GetODataFilter() { var conditions = new List(); - if (this.CreatedTimeFrom > DateTime.MinValue) { - conditions.Add(TableQuery.GenerateFilterConditionForDate("CreatedTime", QueryComparisons.GreaterThanOrEqual, new DateTimeOffset(this.CreatedTimeFrom))); + conditions.Add($"{nameof(OrchestrationInstanceStatus.CreatedTime)} ge datetime'{this.CreatedTimeFrom:O}'"); } if (this.CreatedTimeTo != default(DateTime) && this.CreatedTimeTo < DateTime.MaxValue) { - conditions.Add(TableQuery.GenerateFilterConditionForDate("CreatedTime", QueryComparisons.LessThanOrEqual, new DateTimeOffset(this.CreatedTimeTo))); + conditions.Add($"{nameof(OrchestrationInstanceStatus.CreatedTime)} le datetime'{this.CreatedTimeTo:O}'"); } if (this.RuntimeStatus != null && this.RuntimeStatus.Any()) { - string runtimeCondition = this.RuntimeStatus - .Select(x => TableQuery.GenerateFilterCondition("RuntimeStatus", QueryComparisons.Equal, x.ToString())) - .Aggregate((a, b) => TableQuery.CombineFilters(a, TableOperators.Or, b)); - if (runtimeCondition.Length > 0) - { - conditions.Add(runtimeCondition); - } + conditions.Add($"{string.Join(" or ", this.RuntimeStatus.Select(x => $"{nameof(OrchestrationInstanceStatus.RuntimeStatus)} eq '{x:G}'"))}"); } - if (this.TaskHubNames != null) + if (this.TaskHubNames != null && this.TaskHubNames.Any()) { - string taskHubCondition = this.TaskHubNames - .Select(x => TableQuery.GenerateFilterCondition("TaskHubName", QueryComparisons.Equal, x.ToString())) - .Aggregate((a, b) => TableQuery.CombineFilters(a, TableOperators.Or, b)); - if (taskHubCondition.Count() != 0) - { - conditions.Add(taskHubCondition); - } + conditions.Add($"{string.Join(" or ", this.TaskHubNames.Select(x => $"TaskHubName eq '{x}'"))}"); } if (!string.IsNullOrEmpty(this.InstanceIdPrefix)) @@ -154,26 +134,22 @@ string GetConditions() string greaterThanPrefix = sanitizedPrefix.Substring(0, length) + incrementedLastChar; - conditions.Add(TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, sanitizedPrefix), - TableOperators.And, - TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThan, greaterThanPrefix))); + conditions.Add($"{nameof(OrchestrationInstanceStatus.PartitionKey)} ge '{sanitizedPrefix}'"); + conditions.Add($"{nameof(OrchestrationInstanceStatus.PartitionKey)} lt '{greaterThanPrefix}'"); } if (this.InstanceId != null) { string sanitizedInstanceId = KeySanitation.EscapePartitionKey(this.InstanceId); - conditions.Add(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, sanitizedInstanceId)); + conditions.Add($"{nameof(OrchestrationInstanceStatus.PartitionKey)} eq '{sanitizedInstanceId}'"); } - if (conditions.Count == 0) + return conditions.Count switch { - return string.Empty; - } - - return conditions.Count == 1 ? - conditions[0] : - conditions.Aggregate((a, b) => TableQuery.CombineFilters(a, TableOperators.And, b)); + 0 => null, + 1 => conditions[0], + _ => string.Join(" and ", conditions.Select(c => $"({c})")), + }; } /// diff --git a/src/DurableTask.AzureStorage/Tracking/TableEntityConverter.cs b/src/DurableTask.AzureStorage/Tracking/TableEntityConverter.cs index f50b2da32..f8b1cf715 100644 --- a/src/DurableTask.AzureStorage/Tracking/TableEntityConverter.cs +++ b/src/DurableTask.AzureStorage/Tracking/TableEntityConverter.cs @@ -17,93 +17,209 @@ namespace DurableTask.AzureStorage.Tracking using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; + using System.Linq.Expressions; using System.Reflection; using System.Runtime.Serialization; - using Microsoft.WindowsAzure.Storage.Table; - using Newtonsoft.Json; + using Azure.Data.Tables; /// - /// Utility class for converting [DataContract] objects into DynamicTableEntity and back. + /// Utility class for converting objects that use into TableEntity and back. /// This class makes heavy use of reflection to build the entity converters. /// /// - /// This class is safe for concurrent usage by multiple threads. + /// This class is thread-safe. /// class TableEntityConverter { - readonly ConcurrentDictionary> converterCache; + static readonly ConcurrentDictionary> DeserializeCache = new ConcurrentDictionary>(); + static readonly ConcurrentDictionary> SerializerCache = new ConcurrentDictionary>(); - public TableEntityConverter() - { - this.converterCache = new ConcurrentDictionary>(); - } + public static TableEntity Serialize(object obj) => + SerializerCache.GetOrAdd(obj?.GetType(), t => CreateTableEntitySerializer(t)).Invoke(obj); + + public static object Deserialize(TableEntity entity, Type type) => + DeserializeCache.GetOrAdd(type, t => CreateTableEntityDeserializer(t)).Invoke(entity); - /// - /// Converts a data contract object into a . - /// - public DynamicTableEntity ConvertToTableEntity(object obj) + private static Func CreateTableEntityDeserializer(Type type) { - if (obj == null) + if (type == null) { - throw new ArgumentNullException(nameof(obj)); + throw new ArgumentNullException(nameof(type)); } - Debug.Assert(obj.GetType().GetCustomAttribute() != null); + Debug.Assert(type.GetCustomAttribute() != null); + + MethodInfo getStringMethod = typeof(TableEntity).GetMethod(nameof(TableEntity.GetString), new Type[] { typeof(string) }); + MethodInfo parseMethod = typeof(Enum).GetMethod(nameof(Enum.Parse), new Type[] { typeof(Type), typeof(string) }); + MethodInfo deserializeMethod = typeof(Utils).GetMethod(nameof(Utils.DeserializeFromJson), new Type[] { typeof(string) }); + + ParameterExpression tableParam = Expression.Variable(typeof(TableEntity), "table"); + ParameterExpression outputVar = Expression.Parameter(type, "output"); + var variables = new List { outputVar }; + var body = new List(); - IReadOnlyList propertyConverters = this.converterCache.GetOrAdd( - obj.GetType(), - GetPropertyConvertersForType); + #region if (table == null) throw new ArgumentNullException(nameof(table)); + body.Add(Expression.IfThen( + Expression.Equal(tableParam, Expression.Constant(null, typeof(TableEntity))), + Expression.Throw( + Expression.New( + typeof(ArgumentNullException).GetConstructor(new Type[] { typeof(string) }), + Expression.Constant(tableParam.Name, typeof(string)))))); + #endregion - var tableEntity = new DynamicTableEntity(); - foreach (PropertyConverter propertyConverter in propertyConverters) + #region output = ()FormatterServices.GetUninitializedObject(typeof()); + MethodInfo getUninitializedObjectMethod = typeof(FormatterServices).GetMethod(nameof(FormatterServices.GetUninitializedObject), new Type[] { typeof(Type) }); + body.Add(Expression.Assign( + outputVar, + Expression.Convert( + Expression.Call(null, getUninitializedObjectMethod, Expression.Constant(type, typeof(Type))), + type))); + #endregion + + foreach ((string propertyName, Type memberType, MemberInfo metadata) in EnumerateMembers(type)) { - tableEntity.Properties[propertyConverter.PropertyName] = propertyConverter.GetEntityProperty(obj); + #region output. = /* ... */ + Expression valueExpr; + if (memberType.IsEnum) + { + #region string Variable = table.GetString(""); + ParameterExpression enumStringVar = Expression.Parameter(typeof(string), propertyName + "Variable"); + variables.Add(enumStringVar); + + body.Add(Expression.Assign(enumStringVar, Expression.Call(tableParam, getStringMethod, Expression.Constant(propertyName, typeof(string))))); + #endregion + + #region output. = Variable == null ? default() : ()Enum.Parse(typeof(), Variable); + valueExpr = Expression.Condition( + Expression.Equal(enumStringVar, Expression.Constant(null, typeof(string))), + Expression.Default(memberType), + Expression.Convert(Expression.Call(null, parseMethod, Expression.Constant(memberType, typeof(Type)), enumStringVar), memberType)); + #endregion + } + else if(IsSupportedType(memberType)) + { + #region output. = table.Get(""); + MethodInfo accessorMethod = GetEntityAccessor(memberType); + valueExpr = Expression.Call(tableParam, accessorMethod, Expression.Constant(propertyName, typeof(string))); + #endregion + + if (memberType.IsValueType && !memberType.IsGenericType) // Cannot be null + { + #region output. = table.Get("").GetValueOrDefault(); + valueExpr = Expression.Call(valueExpr, typeof(Nullable<>).MakeGenericType(memberType).GetMethod(nameof(Nullable.GetValueOrDefault), Type.EmptyTypes)); + #endregion + } + } + else + { + // Note: We also deserialize nullable enumerations using JSON for backwards compatibility + #region string Variable = table.GetString(""); + ParameterExpression jsonVariable = Expression.Parameter(typeof(string), propertyName + "Variable"); + variables.Add(jsonVariable); + + body.Add(Expression.Assign(jsonVariable, Expression.Call(tableParam, getStringMethod, Expression.Constant(propertyName, typeof(string))))); + #endregion + + #region output. = = Variable == null ? default() : Utils.DeserializeFromJson<>(Variable); + valueExpr = Expression.Condition( + Expression.Equal(jsonVariable, Expression.Constant(null, typeof(string))), + Expression.Default(memberType), + Expression.Call(null, deserializeMethod.MakeGenericMethod(memberType), jsonVariable)); + #endregion + } + + body.Add(Expression.Assign(Expression.MakeMemberAccess(outputVar, metadata), valueExpr)); + #endregion } - return tableEntity; + #region return (object)output; + body.Add(type != typeof(object) ? Expression.Convert(outputVar, type) : outputVar); + #endregion + + return Expression.Lambda>(Expression.Block(variables, body), tableParam).Compile(); } - public object ConvertFromTableEntity(DynamicTableEntity tableEntity, Func typeFactory) + private static Func CreateTableEntitySerializer(Type type) { - if (tableEntity == null) + if (type == null) { - throw new ArgumentNullException(nameof(tableEntity)); + throw new ArgumentNullException(nameof(type)); } - if (typeFactory == null) - { - throw new ArgumentNullException(nameof(typeFactory)); - } + Debug.Assert(type.GetCustomAttribute() != null); + + // Indexers use "get_Item" and "set_Item" + MethodInfo setItemMethod = typeof(TableEntity).GetMethod("set_Item", new Type[] { typeof(string), typeof(object) }); + MethodInfo serializeMethod = typeof(Utils).GetMethod(nameof(Utils.SerializeToJson), new Type[] { typeof(string) }); + + ParameterExpression objParam = Expression.Parameter(typeof(object), "obj"); + ParameterExpression inputVar = Expression.Variable(type, "input"); + ParameterExpression tableVar = Expression.Variable(typeof(TableEntity), "table"); + var variables = new List { inputVar, tableVar }; + var body = new List(); + + #region if (obj == null) throw new ArgumentNullException(nameof(obj)); + body.Add(Expression.IfThen( + Expression.Equal(objParam, Expression.Constant(null, typeof(object))), + Expression.Throw( + Expression.New( + typeof(ArgumentNullException).GetConstructor(new Type[] { typeof(string) }), + Expression.Constant(objParam.Name, typeof(string)))))); + #endregion - Type objectType = typeFactory(tableEntity); - object createdObject = FormatterServices.GetUninitializedObject(objectType); + #region input = ()obj; + body.Add(Expression.Assign(inputVar, type != typeof(object) ? Expression.Convert(objParam, type) : objParam)); + #endregion - IReadOnlyList propertyConverters = this.converterCache.GetOrAdd( - objectType, - GetPropertyConvertersForType); + #region TableEntity table = new TableEntity(); + body.Add(Expression.Assign(tableVar, Expression.New(typeof(TableEntity)))); + #endregion - foreach (PropertyConverter propertyConverter in propertyConverters) + foreach ((string propertyName, Type memberType, MemberInfo metadata) in EnumerateMembers(type)) { - // Properties with null values are not actually saved/retrieved by table storage. - EntityProperty entityProperty; - if (tableEntity.Properties.TryGetValue(propertyConverter.PropertyName, out entityProperty)) + #region table[""] = (object)/* ... */ + Expression valueExpr; + MemberExpression memberExpr = Expression.MakeMemberAccess(inputVar, metadata); + if (memberType.IsEnum) { - propertyConverter.SetObjectProperty(createdObject, entityProperty); + #region table[""] = input..ToString("G"); + MethodInfo toStringMethod = memberType.GetMethod(nameof(object.ToString), new Type[] { typeof(string) }); + valueExpr = Expression.Call(memberExpr, toStringMethod, Expression.Constant("G", typeof(string))); + #endregion } + else if (IsSupportedType(memberType)) + { + #region table[""] = input.; + valueExpr = memberExpr; + #endregion + } + else + { + // Note: We also serialize nullable enumerations using JSON for backwards compatibility + #region table[""] = Utils.SerializeToJson(input.); + valueExpr = Expression.Call(null, serializeMethod, memberType != typeof(object) ? Expression.Convert(memberExpr, typeof(object)) : memberExpr); + #endregion + } + + body.Add(Expression.Call(tableVar, setItemMethod, Expression.Constant(propertyName, typeof(string)), Expression.Convert(valueExpr, typeof(object)))); + #endregion } - return createdObject; + #region return table; + body.Add(tableVar); + #endregion + + return Expression.Lambda>(Expression.Block(variables, body), objParam).Compile(); } - static List GetPropertyConvertersForType(Type type) + static IEnumerable<(string Name, Type MemberType, MemberInfo Metadata)> EnumerateMembers(Type type) { - var propertyConverters = new List(); - BindingFlags flags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly; + const BindingFlags Flags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly; // Loop through the type hierarchy to find all [DataMember] attributes which belong to [DataContract] classes. while (type != null && type.GetCustomAttribute() != null) { - foreach (MemberInfo member in type.GetMembers(flags)) + foreach (MemberInfo member in type.GetMembers(Flags)) { DataMemberAttribute dataMember = member.GetCustomAttribute(); if (dataMember == null) @@ -117,201 +233,71 @@ static List GetPropertyConvertersForType(Type type) { throw new InvalidDataContractException("Only fields and properties can be marked as [DataMember]."); } - else if (property != null && (!property.CanWrite || !property.CanRead)) + else if ((property != null && (!property.CanWrite || !property.CanRead)) || (field != null && field.IsInitOnly)) { throw new InvalidDataContractException("[DataMember] properties must be both readable and writeable."); } // Timestamp is a reserved property name in Table Storage, so the name needs to be changed. - string propertyName = dataMember.Name ?? member.Name; - if (string.Equals(propertyName, "Timestamp", StringComparison.OrdinalIgnoreCase)) + string name = dataMember.Name ?? member.Name; + if (string.Equals(name, "Timestamp", StringComparison.OrdinalIgnoreCase)) { - propertyName = "_Timestamp"; + name = "_Timestamp"; } - Func getEntityPropertyFunc; - Action setObjectPropertyFunc; - - Type memberValueType = property != null ? property.PropertyType : field.FieldType; - if (typeof(string).IsAssignableFrom(memberValueType)) - { - if (property != null) - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForString((string)property.GetValue(o)); - setObjectPropertyFunc = (o, e) => property.SetValue(o, e.StringValue); - } - else - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForString((string)field.GetValue(o)); - setObjectPropertyFunc = (o, e) => field.SetValue(o, e.StringValue); - } - } - else if (memberValueType.IsEnum) - { - // Enums are serialized as strings for readability. - if (property != null) - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForString(property.GetValue(o).ToString()); - setObjectPropertyFunc = (o, e) => property.SetValue(o, Enum.Parse(memberValueType, e.StringValue)); - } - else - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForString(field.GetValue(o).ToString()); - setObjectPropertyFunc = (o, e) => field.SetValue(o, Enum.Parse(memberValueType, e.StringValue)); - } - } - else if (typeof(int?).IsAssignableFrom(memberValueType)) - { - if (property != null) - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForInt((int?)property.GetValue(o)); - setObjectPropertyFunc = (o, e) => property.SetValue(o, e.Int32Value); - } - else - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForInt((int?)field.GetValue(o)); - setObjectPropertyFunc = (o, e) => field.SetValue(o, e.Int32Value); - } - } - else if (typeof(long?).IsAssignableFrom(memberValueType)) - { - if (property != null) - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForLong((long?)property.GetValue(o)); - setObjectPropertyFunc = (o, e) => property.SetValue(o, e.Int64Value); - } - else - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForLong((long?)field.GetValue(o)); - setObjectPropertyFunc = (o, e) => field.SetValue(o, e.Int64Value); - } - } - else if (typeof(bool?).IsAssignableFrom(memberValueType)) - { - if (property != null) - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForBool((bool?)property.GetValue(o)); - setObjectPropertyFunc = (o, e) => property.SetValue(o, e.BooleanValue); - } - else - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForBool((bool?)field.GetValue(o)); - setObjectPropertyFunc = (o, e) => field.SetValue(o, e.BooleanValue); - } - } - else if (typeof(DateTime?).IsAssignableFrom(memberValueType)) - { - if (property != null) - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForDateTimeOffset((DateTime?)property.GetValue(o)); - setObjectPropertyFunc = (o, e) => property.SetValue(o, e.DateTime); - } - else - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForDateTimeOffset((DateTime?)field.GetValue(o)); - setObjectPropertyFunc = (o, e) => field.SetValue(o, e.DateTime); - } - } - else if (typeof(DateTimeOffset?).IsAssignableFrom(memberValueType)) - { - if (property != null) - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForDateTimeOffset((DateTimeOffset?)property.GetValue(o)); - setObjectPropertyFunc = (o, e) => property.SetValue(o, e.DateTimeOffsetValue); - } - else - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForDateTimeOffset((DateTimeOffset?)field.GetValue(o)); - setObjectPropertyFunc = (o, e) => field.SetValue(o, e.DateTimeOffsetValue); - } - } - else if (typeof(Guid?).IsAssignableFrom(memberValueType)) - { - if (property != null) - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForGuid((Guid?)property.GetValue(o)); - setObjectPropertyFunc = (o, e) => property.SetValue(o, e.GuidValue); - } - else - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForGuid((Guid?)field.GetValue(o)); - setObjectPropertyFunc = (o, e) => field.SetValue(o, e.GuidValue); - } - } - else if (typeof(double?).IsAssignableFrom(memberValueType)) - { - if (property != null) - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForDouble((double?)property.GetValue(o)); - setObjectPropertyFunc = (o, e) => property.SetValue(o, e.DoubleValue); - } - else - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForDouble((double?)field.GetValue(o)); - setObjectPropertyFunc = (o, e) => field.SetValue(o, e.DoubleValue); - } - } - else if (typeof(byte[]).IsAssignableFrom(memberValueType)) - { - if (property != null) - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForByteArray((byte[])property.GetValue(o)); - setObjectPropertyFunc = (o, e) => property.SetValue(o, e.BinaryValue); - } - else - { - getEntityPropertyFunc = o => EntityProperty.GeneratePropertyForByteArray((byte[])field.GetValue(o)); - setObjectPropertyFunc = (o, e) => field.SetValue(o, e.BinaryValue); - } - } - else // assume a serializeable object - { - getEntityPropertyFunc = o => - { - object value = property != null ? property.GetValue(o) : field.GetValue(o); - string json = value != null ? Utils.SerializeToJson(value) : null; - return EntityProperty.GeneratePropertyForString(json); - }; - - setObjectPropertyFunc = (o, e) => - { - string json = e.StringValue; - object value = json != null ? Utils.DeserializeFromJson(json, memberValueType) : null; - if (property != null) - { - property.SetValue(o, value); - } - else - { - field.SetValue(o, value); - } - }; - } - - propertyConverters.Add(new PropertyConverter(propertyName, getEntityPropertyFunc, setObjectPropertyFunc)); + yield return (name, property != null ? property.PropertyType : field.FieldType, member); } type = type.BaseType; } - - return propertyConverters; } - class PropertyConverter + // TODO: Add support for BinaryData if necessary + static bool IsSupportedType(Type type) => + type == typeof(string) || + type == typeof(byte[]) || + type == typeof(bool) || + type == typeof(bool?) || + type == typeof(DateTime) || + type == typeof(DateTime?) || + type == typeof(DateTimeOffset) || + type == typeof(DateTimeOffset?) || + type == typeof(double) || + type == typeof(double?) || + type == typeof(Guid) || + type == typeof(Guid?) || + type == typeof(int) || + type == typeof(int?) || + type == typeof(long) || + type == typeof(long?); + + static MethodInfo GetEntityAccessor(Type type) { - public PropertyConverter( - string propertyName, - Func toEntityPropertyConverter, - Action toObjectPropertyConverter) - { - this.PropertyName = propertyName; - this.GetEntityProperty = toEntityPropertyConverter; - this.SetObjectProperty = toObjectPropertyConverter; - } + if (type == typeof(string)) + return typeof(TableEntity).GetMethod(nameof(TableEntity.GetString), new Type[] { typeof(string) }); + + if (type == typeof(byte[])) + return typeof(TableEntity).GetMethod(nameof(TableEntity.GetBinary), new Type[] { typeof(string) }); + + if (type == typeof(bool) || type == typeof(bool?)) + return typeof(TableEntity).GetMethod(nameof(TableEntity.GetBoolean), new Type[] { typeof(string) }); + + if (type == typeof(DateTime) || type == typeof(DateTime?)) + return typeof(TableEntity).GetMethod(nameof(TableEntity.GetDateTime), new Type[] { typeof(string) }); + + if (type == typeof(DateTimeOffset) || type == typeof(DateTimeOffset?)) + return typeof(TableEntity).GetMethod(nameof(TableEntity.GetDateTimeOffset), new Type[] { typeof(string) }); + + if (type == typeof(double) || type == typeof(double?)) + return typeof(TableEntity).GetMethod(nameof(TableEntity.GetDouble), new Type[] { typeof(string) }); + + if (type == typeof(Guid) || type == typeof(Guid?)) + return typeof(TableEntity).GetMethod(nameof(TableEntity.GetGuid), new Type[] { typeof(string) }); + + if (type == typeof(int) || type == typeof(int?)) + return typeof(TableEntity).GetMethod(nameof(TableEntity.GetInt32), new Type[] { typeof(string) }); - public string PropertyName { get; private set; } - public Func GetEntityProperty { get; private set; } - public Action SetObjectProperty { get; private set; } + return typeof(TableEntity).GetMethod(nameof(TableEntity.GetInt64), new Type[] { typeof(string) }); } } } diff --git a/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs b/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs index 311b2575e..8a738a55a 100644 --- a/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs +++ b/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs @@ -17,100 +17,93 @@ namespace DurableTask.AzureStorage.Tracking using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; + using Azure; using DurableTask.Core; using DurableTask.Core.History; abstract class TrackingStoreBase : ITrackingStore { - protected static readonly HistoryEvent[] EmptyHistoryEventList = new HistoryEvent[0]; - /// - public abstract Task CreateAsync(); + public abstract Task CreateAsync(CancellationToken cancellationToken = default); /// - public abstract Task DeleteAsync(); + public abstract Task DeleteAsync(CancellationToken cancellationToken = default); /// - public virtual Task ExistsAsync() + public virtual Task ExistsAsync(CancellationToken cancellationToken = default) { throw new NotSupportedException(); } /// - public abstract Task GetHistoryEventsAsync(string instanceId, string expectedExecutionId, CancellationToken cancellationToken = default(CancellationToken)); + public abstract Task GetHistoryEventsAsync(string instanceId, string expectedExecutionId, CancellationToken cancellationToken = default); /// - public virtual Task> RewindHistoryAsync(string instanceId, IList failedLeaves, CancellationToken cancellationToken) + public virtual IAsyncEnumerable RewindHistoryAsync(string instanceId, CancellationToken cancellationToken = default) { throw new NotSupportedException(); } /// - public abstract Task FetchInstanceStatusAsync(string instanceId); + public abstract Task FetchInstanceStatusAsync(string instanceId, CancellationToken cancellationToken = default); /// - public abstract Task> GetStateAsync(string instanceId, bool allExecutions, bool fetchInput); - - /// - public abstract Task GetStateAsync(string instanceId, string executionId, bool fetchInput); + public abstract IAsyncEnumerable GetStateAsync(string instanceId, bool allExecutions, bool fetchInput, CancellationToken cancellationToken = default); /// - public virtual Task> GetStateAsync(CancellationToken cancellationToken = default(CancellationToken)) - { - throw new NotSupportedException(); - } + public abstract Task GetStateAsync(string instanceId, string executionId, bool fetchInput, CancellationToken cancellationToken = default); /// - public virtual Task> GetStateAsync(IEnumerable instanceIds) + public virtual IAsyncEnumerable GetStateAsync(CancellationToken cancellationToken = default) { throw new NotSupportedException(); } /// - public virtual Task> GetStateAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, CancellationToken cancellationToken = default(CancellationToken)) + public virtual IAsyncEnumerable GetStateAsync(IEnumerable instanceIds, CancellationToken cancellationToken = default) { throw new NotSupportedException(); } /// - public virtual Task GetStateAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, int top, string continuationToken, CancellationToken cancellationToken = default(CancellationToken)) + public virtual AsyncPageable GetStateAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, CancellationToken cancellationToken = default) { throw new NotSupportedException(); } - public virtual Task GetStateAsync(OrchestrationInstanceStatusQueryCondition condition, int top, string continuationToken, CancellationToken cancellationToken = default(CancellationToken)) + public virtual AsyncPageable GetStateAsync(OrchestrationInstanceStatusQueryCondition condition, CancellationToken cancellationToken = default) { throw new NotSupportedException(); } /// - public abstract Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType); + public abstract Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType, CancellationToken cancellationToken = default); /// - public virtual Task PurgeInstanceHistoryAsync(string instanceId) + public virtual Task PurgeInstanceHistoryAsync(string instanceId, CancellationToken cancellationToken = default) { throw new NotSupportedException(); } /// - public virtual Task PurgeInstanceHistoryAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus) + public virtual Task PurgeInstanceHistoryAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable runtimeStatus, CancellationToken cancellationToken = default) { throw new NotSupportedException(); } /// - public abstract Task SetNewExecutionAsync(ExecutionStartedEvent executionStartedEvent, string eTag, string inputStatusOverride); + public abstract Task SetNewExecutionAsync(ExecutionStartedEvent executionStartedEvent, ETag? eTag, string inputStatusOverride, CancellationToken cancellationToken = default); /// - public virtual Task UpdateStatusForRewindAsync(string instanceId) + public virtual Task UpdateStatusForRewindAsync(string instanceId, CancellationToken cancellationToken = default) { throw new NotSupportedException(); } /// - public abstract Task StartAsync(); + public abstract Task StartAsync(CancellationToken cancellationToken = default); /// - public abstract Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, string eTag); + public abstract Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, ETag? eTag, CancellationToken cancellationToken = default); } } diff --git a/test/DurableTask.AzureServiceFabric.Integration.Tests/FunctionalTests.cs b/test/DurableTask.AzureServiceFabric.Integration.Tests/FunctionalTests.cs index 870854905..ce28f700e 100644 --- a/test/DurableTask.AzureServiceFabric.Integration.Tests/FunctionalTests.cs +++ b/test/DurableTask.AzureServiceFabric.Integration.Tests/FunctionalTests.cs @@ -15,7 +15,6 @@ namespace DurableTask.AzureServiceFabric.Integration.Tests { using System; using System.Collections.Generic; - using System.Text; using System.Threading.Tasks; using DurableTask.AzureServiceFabric.Exceptions; using DurableTask.Core; diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageOrchestrationServiceSettingsTest.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageOrchestrationServiceSettingsTest.cs index b25101ee7..a63a7b332 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageOrchestrationServiceSettingsTest.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageOrchestrationServiceSettingsTest.cs @@ -13,11 +13,6 @@ namespace DurableTask.AzureStorage.Tests { - using System; - using System.Collections.Generic; - using System.Linq; - using System.Text; - using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] @@ -41,7 +36,7 @@ public void TrackingStoreStorageAccountDetailsHasSet() { TaskHubName = "foo", TrackingStoreNamePrefix = "bar", - TrackingStoreStorageAccountDetails = new StorageAccountDetails() + TrackingServiceClientProvider = StorageServiceClientProvider.ForTable("connectionString"), }; Assert.AreEqual("barHistory", settings.HistoryTableName); Assert.AreEqual("barInstances", settings.InstanceTableName); diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScaleTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScaleTests.cs index dcc729fc7..7884c2e32 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScaleTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScaleTests.cs @@ -19,6 +19,10 @@ namespace DurableTask.AzureStorage.Tests using System.Linq; using System.Threading; using System.Threading.Tasks; + using Azure.Data.Tables; + using Azure.Storage.Blobs; + using Azure.Storage.Blobs.Models; + using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Messaging; using DurableTask.AzureStorage.Monitoring; using DurableTask.AzureStorage.Partitioning; @@ -27,9 +31,6 @@ namespace DurableTask.AzureStorage.Tests using DurableTask.Core; using DurableTask.Core.History; using Microsoft.VisualStudio.TestTools.UnitTesting; - using Microsoft.WindowsAzure.Storage; - using Microsoft.WindowsAzure.Storage.Blob; - using Microsoft.WindowsAzure.Storage.Table; /// /// Validates the following requirements: @@ -64,16 +65,15 @@ public async Task DeleteTaskHub() bool useLegacyPartitionManagement = false) { string storageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(); - var storageAccount = CloudStorageAccount.Parse(storageConnectionString); string taskHubName = testName; var settings = new AzureStorageOrchestrationServiceSettings { - TaskHubName = taskHubName, - StorageConnectionString = storageConnectionString, - WorkerId = workerId, AppName = testName, + StorageAccountClientProvider = new StorageAccountClientProvider(storageConnectionString), + TaskHubName = taskHubName, UseLegacyPartitionManagement = useLegacyPartitionManagement, + WorkerId = workerId, }; Trace.TraceInformation($"Task Hub name: {taskHubName}"); @@ -115,11 +115,11 @@ public async Task DeleteTaskHub() { } string expectedContainerName = taskHubName.ToLowerInvariant() + "-leases"; - CloudBlobContainer taskHubContainer = storageAccount.CreateCloudBlobClient().GetContainerReference(expectedContainerName); + BlobContainerClient taskHubContainer = new BlobServiceClient(storageConnectionString).GetBlobContainerClient(expectedContainerName); Assert.IsTrue(await taskHubContainer.ExistsAsync(), $"Task hub blob container {expectedContainerName} was not created."); // Task Hub config blob - CloudBlob infoBlob = taskHubContainer.GetBlobReference("taskhub.json"); + BlobClient infoBlob = taskHubContainer.GetBlobClient("taskhub.json"); Assert.IsTrue(await infoBlob.ExistsAsync(), $"The blob {infoBlob.Name} was not created."); // Task Hub lease container @@ -157,49 +157,19 @@ public async Task DeleteTaskHub() return service; } - private async Task EnsureLeasesMatchControlQueue(string directoryReference, CloudBlobContainer taskHubContainer, ControlQueue[] controlQueues) + private async Task EnsureLeasesMatchControlQueue(string directoryReference, BlobContainerClient taskHubContainer, ControlQueue[] controlQueues) { - CloudBlobDirectory leaseDirectory = taskHubContainer.GetDirectoryReference(directoryReference); - IListBlobItem[] leaseBlobs = (await this.ListBlobsAsync(leaseDirectory)).ToArray(); + BlobItem[] leaseBlobs = await taskHubContainer.GetBlobsAsync(prefix: directoryReference).ToArrayAsync(); Assert.AreEqual(controlQueues.Length, leaseBlobs.Length, "Expected to see the same number of control queues and lease blobs."); - foreach (IListBlobItem blobItem in leaseBlobs) + foreach (BlobItem blobItem in leaseBlobs) { - string path = blobItem.Uri.AbsolutePath; + string path = taskHubContainer.GetBlobClient(blobItem.Name).Uri.AbsolutePath; Assert.IsTrue( controlQueues.Where(q => path.Contains(q.Name)).Any(), $"Could not find any known control queue name in the lease name {path}"); } } - public async Task> ListBlobsAsync(CloudBlobDirectory client) - { - BlobContinuationToken continuationToken = null; - var results = new List(); - do - { - BlobResultSegment response = await TimeoutHandler.ExecuteWithTimeout( - "ListBobs", - "dummyName", - new AzureStorageOrchestrationServiceSettings(), - (context, timeoutToken) => - { - return client.ListBlobsSegmentedAsync( - useFlatBlobListing: true, - blobListingDetails: BlobListingDetails.Metadata, - maxResults: null, - currentToken: continuationToken, - options: null, - operationContext: context, - cancellationToken: timeoutToken); - }); - - continuationToken = response.ContinuationToken; - results.AddRange(response.Results); - } - while (continuationToken != null); - return results; - } - /// /// REQUIREMENT: Workers can be added or removed at any time and control-queue partitions are load-balanced automatically. /// REQUIREMENT: No two workers will ever process the same control queue. @@ -263,12 +233,16 @@ public async Task MultiWorkerLeaseMovement(bool useLegacyPartitionManagement, in $"Blob: {lease.Name}, Owner: {lease.Owner}")); isBalanced = false; - var workersWithLeases = leases.GroupBy(l => l.Owner).ToArray(); - if (workersWithLeases.Count() == currentWorkerCount) + var workersWithLeases = leases + .GroupBy(l => l.Owner) + .Select(x => x.ToArray()) + .ToArray(); + + if (workersWithLeases.Length == currentWorkerCount) { - int maxLeaseCount = workersWithLeases.Max(owned => owned.Count()); - int minLeaseCount = workersWithLeases.Min(owned => owned.Count()); - int totalLeaseCount = workersWithLeases.Sum(owned => owned.Count()); + int maxLeaseCount = workersWithLeases.Max(owned => owned.Length); + int minLeaseCount = workersWithLeases.Min(owned => owned.Length); + int totalLeaseCount = workersWithLeases.Sum(owned => owned.Length); isBalanced = maxLeaseCount - minLeaseCount <= 1 && totalLeaseCount == 4; if (isBalanced) @@ -333,11 +307,11 @@ public async Task TestInstanceAndMessageDistribution() // Create a service and enqueue N messages. // Make sure each partition has messages in it. - var settings = new AzureStorageOrchestrationServiceSettings() + var settings = new AzureStorageOrchestrationServiceSettings { - StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(), - TaskHubName = nameof(TestInstanceAndMessageDistribution), PartitionCount = 4, + StorageAccountClientProvider = new StorageAccountClientProvider(TestHelpers.GetTestStorageAccountConnectionString()), + TaskHubName = nameof(TestInstanceAndMessageDistribution), }; var service = new AzureStorageOrchestrationService(settings); @@ -360,8 +334,7 @@ public async Task TestInstanceAndMessageDistribution() foreach (ControlQueue cloudQueue in controlQueues) { - await cloudQueue.InnerQueue.FetchAttributesAsync(); - int messageCount = cloudQueue.InnerQueue.ApproximateMessageCount.GetValueOrDefault(-1); + int messageCount = await cloudQueue.InnerQueue.GetApproximateMessagesCountAsync(); Trace.TraceInformation($"Queue {cloudQueue.Name} has {messageCount} message(s)."); Assert.IsTrue(messageCount > 0, $"Queue {cloudQueue.Name} didn't receive any messages"); @@ -389,7 +362,7 @@ public async Task TestInstanceAndMessageDistribution() if (tableTrackingStore != null) { - DynamicTableEntity[] entities = (await tableTrackingStore.HistoryTable.ExecuteQueryAsync(new TableQuery())).ReturnedEntities.ToArray(); + TableEntity[] entities = await tableTrackingStore.HistoryTable.ExecuteQueryAsync().ToArrayAsync(); int uniquePartitions = entities.GroupBy(e => e.PartitionKey).Count(); Trace.TraceInformation($"Found {uniquePartitions} unique partition(s) in table storage."); Assert.AreEqual(InstanceCount, uniquePartitions, "Unexpected number of table partitions."); @@ -408,13 +381,13 @@ public async Task TestInstanceAndMessageDistribution() [TestMethod] public async Task PartitionLost_AbandonPrefetchedSession() { - var settings = new AzureStorageOrchestrationServiceSettings() + var settings = new AzureStorageOrchestrationServiceSettings { - PartitionCount = 1, + ControlQueueBufferThreshold = 100, LeaseRenewInterval = TimeSpan.FromMilliseconds(500), + PartitionCount = 1, + StorageAccountClientProvider = new StorageAccountClientProvider(TestHelpers.GetTestStorageAccountConnectionString()), TaskHubName = TestHelpers.GetTestTaskHubName(), - StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(), - ControlQueueBufferThreshold = 100, }; // STEP 1: Start up the service and queue up a large number of messages @@ -445,7 +418,7 @@ public async Task PartitionLost_AbandonPrefetchedSession() // STEP 2: Force the lease to be stolen and wait for the lease status to update. // The orchestration service should detect this and update its state. - BlobLease lease = (await service.ListBlobLeasesAsync()).Single(); + BlobPartitionLease lease = (await service.ListBlobLeasesAsync()).Single(); await lease.Blob.ChangeLeaseAsync( proposedLeaseId: Guid.NewGuid().ToString(), currentLeaseId: lease.Token); @@ -466,7 +439,7 @@ public async Task PartitionLost_AbandonPrefetchedSession() // STEP 4: Verify that all the enqueued messages were abandoned, i.e. put back // onto the queue with their dequeue counts incremented. - IEnumerable queueMessages = + IReadOnlyCollection queueMessages = await controlQueue.InnerQueue.PeekMessagesAsync(settings.ControlQueueBatchSize); Assert.IsTrue(queueMessages.All(msg => msg.DequeueCount == 1)); } @@ -474,16 +447,17 @@ public async Task PartitionLost_AbandonPrefetchedSession() [TestMethod] public async Task MonitorIdleTaskHubDisconnected() { + string connectionString = TestHelpers.GetTestStorageAccountConnectionString(); var settings = new AzureStorageOrchestrationServiceSettings { - StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(), - TaskHubName = nameof(MonitorIdleTaskHubDisconnected), PartitionCount = 4, + StorageAccountClientProvider = new StorageAccountClientProvider(connectionString), + TaskHubName = nameof(MonitorIdleTaskHubDisconnected), UseAppLease = false, }; var service = new AzureStorageOrchestrationService(settings); - var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName); + var monitor = new DisconnectedPerformanceMonitor(connectionString, settings.TaskHubName); await service.DeleteAsync(); @@ -527,16 +501,17 @@ public async Task MonitorIdleTaskHubDisconnected() [TestMethod] public async Task UpdateTaskHubJsonWithNewPartitionCount() { + string connectionString = TestHelpers.GetTestStorageAccountConnectionString(); var settings = new AzureStorageOrchestrationServiceSettings { - StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(), - TaskHubName = nameof(UpdateTaskHubJsonWithNewPartitionCount), PartitionCount = 4, + StorageAccountClientProvider = new StorageAccountClientProvider(connectionString), + TaskHubName = nameof(UpdateTaskHubJsonWithNewPartitionCount), UseAppLease = false, }; var service = new AzureStorageOrchestrationService(settings); - var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName); + var monitor = new DisconnectedPerformanceMonitor(connectionString, settings.TaskHubName); // Empty the existing task hub to make sure we are starting with a clean state. await service.DeleteAsync(); @@ -603,17 +578,18 @@ public async Task UpdateTaskHubJsonWithNewPartitionCount() [TestMethod] public async Task MonitorIncreasingControlQueueLoadDisconnected() { - var settings = new AzureStorageOrchestrationServiceSettings() + string connectionString = TestHelpers.GetTestStorageAccountConnectionString(); + var settings = new AzureStorageOrchestrationServiceSettings { - StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(), - TaskHubName = nameof(MonitorIncreasingControlQueueLoadDisconnected), PartitionCount = 4, + StorageAccountClientProvider = new StorageAccountClientProvider(connectionString), + TaskHubName = nameof(MonitorIncreasingControlQueueLoadDisconnected), UseAppLease = false, }; var service = new AzureStorageOrchestrationService(settings); - var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName); + var monitor = new DisconnectedPerformanceMonitor(connectionString, settings.TaskHubName); int simulatedWorkerCount = 0; await service.CreateAsync(); diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 95d92be65..8cd1a3682 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -15,7 +15,6 @@ namespace DurableTask.AzureStorage.Tests { using System; using System.Collections.Generic; - using System.Configuration; using System.Diagnostics; using System.IO; using System.Linq; @@ -23,15 +22,16 @@ namespace DurableTask.AzureStorage.Tests using System.Text; using System.Threading; using System.Threading.Tasks; + using Azure.Data.Tables; + using Azure.Storage.Blobs; + using Azure.Storage.Blobs.Models; + using DurableTask.AzureStorage.Storage; using DurableTask.AzureStorage.Tracking; using DurableTask.Core; using DurableTask.Core.Exceptions; using DurableTask.Core.History; using Microsoft.Practices.EnterpriseLibrary.SemanticLogging.Utility; using Microsoft.VisualStudio.TestTools.UnitTesting; - using Microsoft.WindowsAzure.Storage; - using Microsoft.WindowsAzure.Storage.Blob; - using Microsoft.WindowsAzure.Storage.Table; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -168,34 +168,6 @@ public async Task GetAllOrchestrationStatuses() } } - [TestMethod] - public async Task GetPaginatedStatuses() - { - using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) - { - // Execute the orchestrator twice. Orchestrator will be replied. However instances might be two. - await host.StartAsync(); - var client = await host.StartOrchestrationAsync(typeof(Orchestrations.SayHelloInline), "world one"); - await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - client = await host.StartOrchestrationAsync(typeof(Orchestrations.SayHelloInline), "world two"); - await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); - - DurableStatusQueryResult queryResult = await host.service.GetOrchestrationStateAsync( - new OrchestrationInstanceStatusQueryCondition(), - 1, - null); - Assert.AreEqual(1, queryResult.OrchestrationState.Count()); - Assert.IsNotNull(queryResult.ContinuationToken); - queryResult = await host.service.GetOrchestrationStateAsync( - new OrchestrationInstanceStatusQueryCondition(), - 1, - queryResult.ContinuationToken); - Assert.AreEqual(1, queryResult.OrchestrationState.Count()); - Assert.IsNull(queryResult.ContinuationToken); - await host.StopAsync(); - } - } - [TestMethod] public async Task GetInstanceIdsByPrefix() { @@ -343,7 +315,7 @@ public async Task PurgeInstanceHistoryForSingleInstanceWithoutLargeMessageBlobs( orchestrationStateList = await client.GetStateAsync(instanceId); Assert.AreEqual(1, orchestrationStateList.Count); - Assert.IsNull(orchestrationStateList.First()); + Assert.IsNull(orchestrationStateList[0]); await host.StopAsync(); } @@ -420,13 +392,12 @@ public async Task PurgeInstanceHistoryForSingleInstanceWithLargeMessageBlobs() await client.PurgeInstanceHistory(); - List historyEventsAfterPurging = await client.GetOrchestrationHistoryAsync(instanceId); Assert.AreEqual(0, historyEventsAfterPurging.Count); orchestrationStateList = await client.GetStateAsync(instanceId); Assert.AreEqual(1, orchestrationStateList.Count); - Assert.IsNull(orchestrationStateList.First()); + Assert.IsNull(orchestrationStateList[0]); blobCount = await this.GetBlobCount("test-largemessages", instanceId); Assert.AreEqual(0, blobCount); @@ -522,19 +493,19 @@ public async Task PurgeInstanceHistoryForTimePeriodDeleteAll() firstOrchestrationStateList = await client.GetStateAsync(firstInstanceId); Assert.AreEqual(1, firstOrchestrationStateList.Count); - Assert.IsNull(firstOrchestrationStateList.First()); + Assert.IsNull(firstOrchestrationStateList[0]); secondOrchestrationStateList = await client.GetStateAsync(secondInstanceId); Assert.AreEqual(1, secondOrchestrationStateList.Count); - Assert.IsNull(secondOrchestrationStateList.First()); + Assert.IsNull(secondOrchestrationStateList[0]); thirdOrchestrationStateList = await client.GetStateAsync(thirdInstanceId); Assert.AreEqual(1, thirdOrchestrationStateList.Count); - Assert.IsNull(thirdOrchestrationStateList.First()); + Assert.IsNull(thirdOrchestrationStateList[0]); fourthOrchestrationStateList = await client.GetStateAsync(fourthInstanceId); Assert.AreEqual(1, fourthOrchestrationStateList.Count); - Assert.IsNull(fourthOrchestrationStateList.First()); + Assert.IsNull(fourthOrchestrationStateList[0]); blobCount = await this.GetBlobCount("test-largemessages", fourthInstanceId); Assert.AreEqual(0, blobCount); @@ -545,45 +516,12 @@ public async Task PurgeInstanceHistoryForTimePeriodDeleteAll() private async Task GetBlobCount(string containerName, string directoryName) { - string storageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(); - CloudStorageAccount storageAccount; - if (!CloudStorageAccount.TryParse(storageConnectionString, out storageAccount)) - { - Assert.Fail("Couldn't find the connection string to use to look up blobs!"); - return 0; - } + var client = new BlobServiceClient(TestHelpers.GetTestStorageAccountConnectionString()); - CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient(); - - CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(containerName); - await cloudBlobContainer.CreateIfNotExistsAsync(); - CloudBlobDirectory instanceDirectory = cloudBlobContainer.GetDirectoryReference(directoryName); - var blobs = new List(); - BlobContinuationToken blobContinuationToken = null; - do - { - BlobResultSegment results = await TimeoutHandler.ExecuteWithTimeout("GetBlobCount", "dummyAccount", new AzureStorageOrchestrationServiceSettings(), (context, timeoutToken) => - { - return instanceDirectory.ListBlobsSegmentedAsync( - useFlatBlobListing: true, - blobListingDetails: BlobListingDetails.Metadata, - maxResults: null, - currentToken: blobContinuationToken, - options: null, - operationContext: context, - cancellationToken: timeoutToken); - }); - - blobContinuationToken = results.ContinuationToken; - blobs.AddRange(results.Results); - } while (blobContinuationToken != null); + var containerClient = client.GetBlobContainerClient(containerName); + await containerClient.CreateIfNotExistsAsync(); - Trace.TraceInformation( - "Found {0} blobs: {1}{2}", - blobs.Count, - Environment.NewLine, - string.Join(Environment.NewLine, blobs.Select(b => b.Uri))); - return blobs.Count; + return await containerClient.GetBlobsAsync(traits: BlobTraits.Metadata, prefix: directoryName).CountAsync(); } @@ -648,7 +586,7 @@ public async Task PurgeInstanceHistoryForTimePeriodDeletePartially() firstOrchestrationStateList = await client.GetStateAsync(firstInstanceId); Assert.AreEqual(1, firstOrchestrationStateList.Count); - Assert.IsNull(firstOrchestrationStateList.First()); + Assert.IsNull(firstOrchestrationStateList[0]); secondOrchestrationStateList = await client.GetStateAsync(secondInstanceId); Assert.AreEqual(1, secondOrchestrationStateList.Count); @@ -822,7 +760,7 @@ public async Task ActorOrchestrationDeleteAllLargeMessageBlobs(bool enableExtend orchestrationStateList = await client.GetStateAsync(instanceId); Assert.AreEqual(1, orchestrationStateList.Count); - Assert.IsNull(orchestrationStateList.First()); + Assert.IsNull(orchestrationStateList[0]); blobCount = await this.GetBlobCount("test-largemessages", instanceId); Assert.AreEqual(0, blobCount); @@ -1548,12 +1486,12 @@ public async Task LargeTableTextMessagePayloads_SizeViolation_BlobUrl(bool enabl var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - await ValidateBlobUrlAsync( + await ValidateLargeMessageBlobUrlAsync( host.TaskHub, client.InstanceId, status?.Input, Encoding.UTF8.GetByteCount(message)); - await ValidateBlobUrlAsync( + await ValidateLargeMessageBlobUrlAsync( host.TaskHub, client.InstanceId, status?.Output, @@ -1581,12 +1519,12 @@ public async Task LargeOverallTextMessagePayloads_BlobUrl(bool enableExtendedSes var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - await ValidateBlobUrlAsync( + await ValidateLargeMessageBlobUrlAsync( host.TaskHub, client.InstanceId, status?.Input, Encoding.UTF8.GetByteCount(message)); - await ValidateBlobUrlAsync( + await ValidateLargeMessageBlobUrlAsync( host.TaskHub, client.InstanceId, status?.Output, @@ -2060,31 +1998,31 @@ public async Task DoubleFanOut(bool enableExtendedSessions) } } - private static async Task ValidateBlobUrlAsync(string taskHubName, string instanceId, string value, int originalPayloadSize = 0) + private static async Task ValidateLargeMessageBlobUrlAsync(string taskHubName, string instanceId, string value, int originalPayloadSize = 0) { string sanitizedInstanceId = KeySanitation.EscapePartitionKey(instanceId); - CloudStorageAccount account = CloudStorageAccount.Parse(TestHelpers.GetTestStorageAccountConnectionString()); - Assert.IsTrue(value.StartsWith(account.BlobStorageUri.PrimaryUri.OriginalString)); + var serviceClient = new BlobServiceClient(TestHelpers.GetTestStorageAccountConnectionString()); + Assert.IsTrue(value.StartsWith(serviceClient.Uri.OriginalString)); Assert.IsTrue(value.Contains("/" + sanitizedInstanceId + "/")); Assert.IsTrue(value.EndsWith(".json.gz")); + string blobName = value.Split('/').Last(); + Assert.IsTrue(await new Blob(serviceClient, new Uri(value)).ExistsAsync(), $"Blob named {blobName} is expected to exist."); + string containerName = $"{taskHubName.ToLowerInvariant()}-largemessages"; - CloudBlobClient client = account.CreateCloudBlobClient(); - CloudBlobContainer container = client.GetContainerReference(containerName); + BlobContainerClient container = serviceClient.GetBlobContainerClient(containerName); Assert.IsTrue(await container.ExistsAsync(), $"Blob container {containerName} is expected to exist."); - - await client.GetBlobReferenceFromServerAsync(new Uri(value)); - CloudBlobDirectory instanceDirectory = container.GetDirectoryReference(sanitizedInstanceId); - - string blobName = value.Split('/').Last(); - CloudBlob blob = instanceDirectory.GetBlobReference(blobName); - Assert.IsTrue(await blob.ExistsAsync(), $"Blob named {blob.Uri} is expected to exist."); + BlobItem blob = await container + .GetBlobsByHierarchyAsync(BlobTraits.Metadata, prefix: sanitizedInstanceId) + .Where(x => x.IsBlob && x.Blob.Name == sanitizedInstanceId + "/" + blobName) + .Select(x => x.Blob) + .SingleOrDefaultAsync(); + Assert.IsNotNull(blob); if (originalPayloadSize > 0) { - await blob.FetchAttributesAsync(); - Assert.IsTrue(blob.Properties.Length < originalPayloadSize, "Blob is expected to be compressed"); + Assert.IsTrue(blob.Properties.ContentLength < originalPayloadSize, "Blob is expected to be compressed"); } } @@ -2997,7 +2935,7 @@ protected override string Execute(TaskContext context, string input) if (ShouldFail) { - throw new Exception("Simulating unhandled activty function failure..."); + throw new Exception("Simulating unhandled activity function failure..."); } return $"Hello, {input}!"; @@ -3017,7 +2955,7 @@ protected override string Execute(TaskContext context, string input) if (ShouldFail1 || ShouldFail2) //&& (input == "0" || input == "2")) { - throw new Exception("Simulating unhandled activty function failure..."); + throw new Exception("Simulating unhandled activity function failure..."); } return $"Hello, {input}!"; @@ -3037,7 +2975,7 @@ protected override string Execute(TaskContext context, string input) if (ShouldFail1 || ShouldFail2) { - throw new Exception("Simulating unhandled activty function failure..."); + throw new Exception("Simulating unhandled activity function failure..."); } return $"Hello, {input}!"; @@ -3057,7 +2995,7 @@ protected override string Execute(TaskContext context, string input) if (ShouldFail1 || ShouldFail2) { - throw new Exception("Simulating unhandled activty function failure..."); + throw new Exception("Simulating unhandled activity function failure..."); } return $"Hello, {input}!"; @@ -3077,7 +3015,7 @@ protected override string Execute(TaskContext context, string input) if (ShouldFail1 || ShouldFail2) { - throw new Exception("Simulating unhandled activty function failure..."); + throw new Exception("Simulating unhandled activity function failure..."); } return $"Hello, {input}!"; @@ -3161,16 +3099,16 @@ protected override string Execute(TaskContext context, string message) internal class WriteTableRow : TaskActivity, string> { - static CloudTable cachedTable; + static TableClient cachedTable; - internal static CloudTable TestCloudTable + internal static TableClient TestTable { get { if (cachedTable == null) { string connectionString = TestHelpers.GetTestStorageAccountConnectionString(); - CloudTable table = CloudStorageAccount.Parse(connectionString).CreateCloudTableClient().GetTableReference("TestTable"); + TableClient table = new TableServiceClient(connectionString).GetTableClient("TestTable"); table.CreateIfNotExistsAsync().Wait(); cachedTable = table; } @@ -3181,10 +3119,10 @@ internal static CloudTable TestCloudTable protected override string Execute(TaskContext context, Tuple rowData) { - var entity = new DynamicTableEntity( + var entity = new TableEntity( partitionKey: rowData.Item1, rowKey: $"{rowData.Item2}.{Guid.NewGuid():N}"); - TestCloudTable.ExecuteAsync(TableOperation.Insert(entity)).Wait(); + TestTable.AddEntityAsync(entity).Wait(); return null; } } @@ -3193,13 +3131,7 @@ internal class CountTableRows : TaskActivity { protected override int Execute(TaskContext context, string partitionKey) { - var query = new TableQuery().Where( - TableQuery.GenerateFilterCondition( - "PartitionKey", - QueryComparisons.Equal, - partitionKey)); - - return WriteTableRow.TestCloudTable.ExecuteQuerySegmentedAsync(query, new TableContinuationToken()).Result.Count(); + return WriteTableRow.TestTable.Query(filter: $"PartitionKey eq '{partitionKey}'").Count(); } } internal class Echo : TaskActivity diff --git a/test/DurableTask.AzureStorage.Tests/AzureTableTrackingStoreTest.cs b/test/DurableTask.AzureStorage.Tests/AzureTableTrackingStoreTest.cs index 04b74fc22..f5f43180d 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureTableTrackingStoreTest.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureTableTrackingStoreTest.cs @@ -15,19 +15,18 @@ namespace DurableTask.AzureStorage.Tests { using System; using System.Collections.Generic; + using System.Globalization; using System.Linq; - using System.Text; using System.Threading; using System.Threading.Tasks; + using Azure; + using Azure.Data.Tables; using DurableTask.AzureStorage.Monitoring; using DurableTask.AzureStorage.Storage; using DurableTask.AzureStorage.Tracking; using DurableTask.Core; using Microsoft.VisualStudio.TestTools.UnitTesting; - using Microsoft.WindowsAzure.Storage; - using Microsoft.WindowsAzure.Storage.Table; using Moq; - using Newtonsoft.Json; [TestClass] public class AzureTableTrackingStoreTest @@ -35,203 +34,82 @@ public class AzureTableTrackingStoreTest [TestMethod] public async Task QueryStatus_WithContinuationToken_NoInputToken() { - var fixture = new QueryFixture(); - fixture.SetUpQueryStateWithPagerWithoutInputToken(); + const string TableName = "MockTable"; + const string ConnectionString = "UseDevelopmentStorage=true"; - var inputState = new List(); - inputState.Add(OrchestrationStatus.Running); - inputState.Add(OrchestrationStatus.Completed); - inputState.Add(OrchestrationStatus.Failed); + using var tokenSource = new CancellationTokenSource(); - var result = await fixture.TrackingStore.GetStateAsync(fixture.ExpectedCreatedDateFrom, fixture.ExpectedCreatedDateTo, inputState, 3, fixture.InputToken); - - Assert.AreEqual("", fixture.ActualPassedTokenString); - - Assert.AreEqual(fixture.ExpectedResult.ContinuationToken, result.ContinuationToken); - Assert.AreEqual(fixture.ExpectedResult.OrchestrationState.Count(), result.OrchestrationState.Count()); - Assert.AreEqual(fixture.ExpectedResult.OrchestrationState.FirstOrDefault().Name, result.OrchestrationState.FirstOrDefault().Name); - fixture.VerifyQueryStateWithPager(); - } - - [TestMethod] - public async Task QueryStatus_WithContinuationToken_InputToken() - { - var fixture = new QueryFixture(); - var inputToken = new TableContinuationToken() + var settings = new AzureStorageOrchestrationServiceSettings { - NextPartitionKey = "qux", - NextRowKey = "quux", - NextTableName = "corge", + StorageAccountClientProvider = new StorageAccountClientProvider(ConnectionString), }; - var inputTokenString = JsonConvert.SerializeObject(inputToken); - - fixture.SetupQueryStateWithPagerWithInputToken(inputTokenString); - - var inputState = new List(); - inputState.Add(OrchestrationStatus.Running); - inputState.Add(OrchestrationStatus.Completed); - inputState.Add(OrchestrationStatus.Failed); - - var result = await fixture.TrackingStore.GetStateAsync(fixture.ExpectedCreatedDateFrom, fixture.ExpectedCreatedDateTo, inputState, 3, fixture.InputToken); - - Assert.AreEqual(inputTokenString, fixture.ActualPassedTokenString); - - Assert.AreEqual(fixture.ExpectedResult.ContinuationToken, result.ContinuationToken); - Assert.AreEqual(fixture.ExpectedResult.OrchestrationState.Count(), result.OrchestrationState.Count()); - Assert.AreEqual(fixture.ExpectedResult.OrchestrationState.FirstOrDefault().Name, result.OrchestrationState.FirstOrDefault().Name); - fixture.VerifyQueryStateWithPager(); - } - - private class QueryFixture - { - private readonly Mock tableMock; - - public AzureTableTrackingStore TrackingStore { get; set; } - - public Table TableMock => this.tableMock.Object; - - public DateTime ExpectedCreatedDateFrom { get; set; } - - public DateTime ExpectedCreatedDateTo { get; set; } - - public int ExpectedTop { get; set; } - - public DurableStatusQueryResult ExpectedResult { get; set; } - - public string ExpectedNextPartitionKey { get; set; } - - public string ExpectedTokenObject { get; set; } - - public string InputToken { get; set; } - - public string ExpectedPassedTokenObject { get; set; } - - public string ActualPassedTokenString { get; set; } - - public List InputStatus { get; set; } - - public List InputState { get; set; } - - public QueryFixture() - { - var azureStorageClient = new AzureStorageClient(new AzureStorageOrchestrationServiceSettings() { StorageConnectionString = "UseDevelopmentStorage=true"}); - var cloudStorageAccount = CloudStorageAccount.Parse(azureStorageClient.Settings.StorageConnectionString); - var cloudTableClient = cloudStorageAccount.CreateCloudTableClient(); - - this.tableMock = new Mock
(azureStorageClient, cloudTableClient, "MockTable"); - } - - private void SetUpQueryStateWithPager(string inputToken) - { - this.ExpectedCreatedDateFrom = DateTime.UtcNow; - this.ExpectedCreatedDateTo = DateTime.UtcNow; - this.ExpectedTop = 3; - - this.InputToken = inputToken; - SetupQueryStateWithPagerInputStatus(); - SetUpQueryStateWithPagerResult(); - SetupExecuteQuerySegmentMock(); - SetupTrackingStore(); - } + var azureStorageClient = new AzureStorageClient(settings); + var tableServiceClient = new Mock(MockBehavior.Strict, ConnectionString); + var tableClient = new Mock(MockBehavior.Strict, ConnectionString, TableName); + tableClient.Setup(t => t.Name).Returns(TableName); + tableServiceClient.Setup(t => t.GetTableClient(TableName)).Returns(tableClient.Object); - public void SetUpQueryStateWithPagerWithoutInputToken() - { - SetUpQueryStateWithPager(""); - } + var table = new Table(azureStorageClient, tableServiceClient.Object, TableName); + var stats = new AzureStorageOrchestrationServiceStats(); + var trackingStore = new AzureTableTrackingStore(stats, table); - public void SetupQueryStateWithPagerWithInputToken(string serializedInputToken) + DateTime expectedCreatedDateFrom = DateTime.UtcNow; + DateTime expectedCreatedDateTo = expectedCreatedDateFrom.AddHours(1); + var inputState = new List { - this.ExpectedPassedTokenObject = serializedInputToken; - SetUpQueryStateWithPager(serializedInputToken); - } - - public void VerifyQueryStateWithPager() - { - this.tableMock.Verify(t => t.ExecuteQuerySegmentAsync( - It.IsAny>(), - It.IsAny(), - It.IsAny())); - } - - private void SetupExecuteQuerySegmentMock() - { - var segment = (TableEntitiesResponseInfo)System.Runtime.Serialization.FormatterServices.GetSafeUninitializedObject(typeof(TableEntitiesResponseInfo)); - segment.GetType().GetProperty("ReturnedEntities").SetValue(segment, this.InputStatus); - segment.GetType().GetProperty("ContinuationToken").SetValue(segment, this.ExpectedTokenObject); - - this.tableMock.Setup(t => t.ExecuteQuerySegmentAsync( - It.IsAny>(), - It.IsAny(), - It.IsAny())) - .Returns(Task.FromResult(segment)) - .Callback, CancellationToken, string>( - (q, cancelToken, token) => - { - this.ActualPassedTokenString = token; - Assert.AreEqual(this.ExpectedTop, q.TakeCount); - }); - } - - private void SetupQueryStateWithPagerInputStatus() + OrchestrationStatus.Running, + OrchestrationStatus.Completed, + OrchestrationStatus.Failed, + }; + var expected = new List { - this.InputStatus = new List() + new OrchestrationInstanceStatus { - new OrchestrationInstanceStatus() - { - Name = "foo", - RuntimeStatus = "Running" - }, - new OrchestrationInstanceStatus() - { - Name = "bar", - RuntimeStatus = "Completed" - }, - new OrchestrationInstanceStatus() - { - Name = "baz", - RuntimeStatus = "Failed" - } - }; - } - - private void SetUpQueryStateWithPagerResult() - { - this.ExpectedResult = new DurableStatusQueryResult(); - - this.ExpectedNextPartitionKey = "foo"; - var token = new TableContinuationToken() + Name = "foo", + RuntimeStatus = "Running" + }, + new OrchestrationInstanceStatus { - NextPartitionKey = ExpectedNextPartitionKey, - NextRowKey = "bar", - NextTableName = "baz", - }; - this.ExpectedTokenObject = JsonConvert.SerializeObject(token); - this.ExpectedResult.ContinuationToken = this.ExpectedTokenObject; - this.ExpectedResult.OrchestrationState = new List() + Name = "bar", + RuntimeStatus = "Completed" + }, + new OrchestrationInstanceStatus { - new OrchestrationState() - { - Name = "foo", - OrchestrationStatus = OrchestrationStatus.Running - }, - new OrchestrationState() - { - Name = "bar", - OrchestrationStatus = OrchestrationStatus.Completed - }, - new OrchestrationState() + Name = "baz", + RuntimeStatus = "Failed" + } + }; + + string expectedFilter = string.Format( + CultureInfo.InvariantCulture, + "({0} ge datetime'{1:O}') and ({0} le datetime'{2:O}') and ({3} eq '{4:G}' or {3} eq '{5:G}' or {3} eq '{6:G}')", + nameof(OrchestrationInstanceStatus.CreatedTime), + expectedCreatedDateFrom, + expectedCreatedDateTo, + nameof(OrchestrationInstanceStatus.RuntimeStatus), + OrchestrationStatus.Running, + OrchestrationStatus.Completed, + OrchestrationStatus.Failed); + + tableClient + .Setup(t => t.QueryAsync(expectedFilter, null, null, tokenSource.Token)) + .Returns(AsyncPageable.FromPages( + new[] { - Name = "baz", - OrchestrationStatus = OrchestrationStatus.Failed - } - }; - } + Page.FromValues(expected, null, new Mock().Object) + })); + + // .ExecuteQueryAsync(filter, select, cancellationToken) + var actual = await trackingStore + .GetStateAsync(expectedCreatedDateFrom, expectedCreatedDateTo, inputState, tokenSource.Token) + .ToListAsync(); - private void SetupTrackingStore() + Assert.AreEqual(expected.Count, actual.Count); + for (int i = 0; i < expected.Count; i++) { - var stats = new AzureStorageOrchestrationServiceStats(); - this.TrackingStore = new AzureTableTrackingStore(stats, this.TableMock); + Assert.AreEqual(expected[i].Name, actual[i].Name); + Assert.AreEqual(Enum.Parse(typeof(OrchestrationStatus), expected[i].RuntimeStatus), actual[i].OrchestrationStatus); } } } diff --git a/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj b/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj index 0f133e4ce..c01d2c199 100644 --- a/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj +++ b/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj @@ -10,14 +10,14 @@ - + - + diff --git a/test/DurableTask.AzureStorage.Tests/OrchestrationInstanceStatusQueryConditionTest.cs b/test/DurableTask.AzureStorage.Tests/OrchestrationInstanceStatusQueryConditionTest.cs index 1f1cf9fa3..995a5bc57 100644 --- a/test/DurableTask.AzureStorage.Tests/OrchestrationInstanceStatusQueryConditionTest.cs +++ b/test/DurableTask.AzureStorage.Tests/OrchestrationInstanceStatusQueryConditionTest.cs @@ -31,8 +31,7 @@ public void OrchestrationInstanceQuery_RuntimeStatus() RuntimeStatus = runtimeStatus }; - var query = condition.ToTableQuery(); - Assert.AreEqual("RuntimeStatus eq 'Running'", query.FilterString); + Assert.AreEqual("RuntimeStatus eq 'Running'", condition.ToOData().Filter); } [TestMethod] @@ -44,10 +43,9 @@ public void OrchestrationInstanceQuery_CreatedTime() CreatedTimeTo = new DateTime(2018, 1, 10, 10, 10, 50, DateTimeKind.Utc) }; - var result = condition.ToTableQuery().FilterString; Assert.AreEqual( "(CreatedTime ge datetime'2018-01-10T10:10:10.0000000Z') and (CreatedTime le datetime'2018-01-10T10:10:50.0000000Z')", - condition.ToTableQuery().FilterString); + condition.ToOData().Filter); } [TestMethod] @@ -60,10 +58,9 @@ public void OrchestrationInstanceQuery_CreatedTimeOnly() RuntimeStatus = new List(), }; - var result = condition.ToTableQuery().FilterString; Assert.AreEqual( "CreatedTime ge datetime'2018-01-10T10:10:10.0000000Z'", - condition.ToTableQuery().FilterString); + condition.ToOData().Filter); } [TestMethod] @@ -76,7 +73,7 @@ public void OrchestrationInstanceQuery_CreatedTimeVariations() Assert.AreEqual( "CreatedTime ge datetime'2018-01-10T10:10:10.0000000Z'", - condition.ToTableQuery().FilterString); + condition.ToOData().Filter); condition = new OrchestrationInstanceStatusQueryCondition { @@ -85,7 +82,7 @@ public void OrchestrationInstanceQuery_CreatedTimeVariations() Assert.AreEqual( "CreatedTime le datetime'2018-01-10T10:10:50.0000000Z'", - condition.ToTableQuery().FilterString); + condition.ToOData().Filter); } [TestMethod] @@ -100,16 +97,15 @@ public void OrchestrationInstanceQuery_Combination() }; Assert.AreEqual( - "((CreatedTime ge datetime'2018-01-10T10:10:10.0000000Z') and (CreatedTime le datetime'2018-01-10T10:10:50.0000000Z')) and (RuntimeStatus eq 'Running')", - condition.ToTableQuery().FilterString); + "(CreatedTime ge datetime'2018-01-10T10:10:10.0000000Z') and (CreatedTime le datetime'2018-01-10T10:10:50.0000000Z') and (RuntimeStatus eq 'Running')", + condition.ToOData().Filter); } [TestMethod] public void OrchestrationInstanceQuery_NoParameter() { var condition = new OrchestrationInstanceStatusQueryCondition(); - var query = condition.ToTableQuery(); - Assert.IsTrue(string.IsNullOrWhiteSpace(query.FilterString)); + Assert.IsTrue(string.IsNullOrWhiteSpace(condition.ToOData().Filter)); } [TestMethod] @@ -125,8 +121,8 @@ public void OrchestrationInstanceQuery_MultipleRuntimeStatus() }; Assert.AreEqual( - "(((CreatedTime ge datetime'2018-01-10T10:10:10.0000000Z') and (CreatedTime le datetime'2018-01-10T10:10:50.0000000Z')) and ((RuntimeStatus eq 'Running') or (RuntimeStatus eq 'Completed'))) and ((TaskHubName eq 'FooProduction') or (TaskHubName eq 'BarStaging'))", - condition.ToTableQuery().FilterString); + "(CreatedTime ge datetime'2018-01-10T10:10:10.0000000Z') and (CreatedTime le datetime'2018-01-10T10:10:50.0000000Z') and (RuntimeStatus eq 'Running' or RuntimeStatus eq 'Completed') and (TaskHubName eq 'FooProduction' or TaskHubName eq 'BarStaging')", + condition.ToOData().Filter); } [TestMethod] @@ -137,7 +133,7 @@ public void OrchestrationInstanceQuery_WithAppId() TaskHubNames = new string[] { "FooProduction" } }; Assert.AreEqual("TaskHubName eq 'FooProduction'", - condition.ToTableQuery().FilterString + condition.ToOData().Filter ); } @@ -152,8 +148,8 @@ public void OrchestrationInstanceQuery_Parse() runtimeStatus); Assert.AreEqual( - "((CreatedTime ge datetime'2018-01-10T10:10:10.0000000Z') and (CreatedTime le datetime'2018-01-10T10:10:50.0000000Z')) and (RuntimeStatus eq 'Running')", - condition.ToTableQuery().FilterString); + "(CreatedTime ge datetime'2018-01-10T10:10:10.0000000Z') and (CreatedTime le datetime'2018-01-10T10:10:50.0000000Z') and (RuntimeStatus eq 'Running')", + condition.ToOData().Filter); } [TestMethod] @@ -162,8 +158,7 @@ public void OrchestrationInstanceQuery_ParseOptional() var runtimeStatus = new List(); runtimeStatus.Add(OrchestrationStatus.Running); var condition = OrchestrationInstanceStatusQueryCondition.Parse(default(DateTime), null, runtimeStatus); - var query = condition.ToTableQuery(); - Assert.AreEqual("RuntimeStatus eq 'Running'", query.FilterString); + Assert.AreEqual("RuntimeStatus eq 'Running'", condition.ToOData().Filter); } [TestMethod] @@ -174,10 +169,9 @@ public void OrchestrationInstanceQuery_InstanceIdPrefix() InstanceIdPrefix = "aaab", }; - var result = condition.ToTableQuery().FilterString; Assert.AreEqual( "(PartitionKey ge 'aaab') and (PartitionKey lt 'aaac')", - condition.ToTableQuery().FilterString); + condition.ToOData().Filter); } [TestMethod] @@ -188,8 +182,7 @@ public void OrchestrationInstanceQuery_InstanceId() InstanceId = "abc123", }; - string result = condition.ToTableQuery().FilterString; - Assert.AreEqual("PartitionKey eq 'abc123'", result); + Assert.AreEqual("PartitionKey eq 'abc123'", condition.ToOData().Filter); } [TestMethod] @@ -200,7 +193,7 @@ public void OrchestrationInstanceQuery_EmptyInstanceId() InstanceId = "", // This is technically legal }; - string result = condition.ToTableQuery().FilterString; + string result = condition.ToOData().Filter; Assert.AreEqual("PartitionKey eq ''", result); } } diff --git a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs index 37073cbb3..32cc7f714 100644 --- a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs +++ b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs @@ -23,7 +23,6 @@ namespace DurableTask.AzureStorage.Tests static class TestHelpers { - public static TestOrchestrationHost GetTestOrchestrationHost( bool enableExtendedSessions, int extendedSessionTimeoutInSeconds = 30, @@ -34,11 +33,11 @@ static class TestHelpers var settings = new AzureStorageOrchestrationServiceSettings { - StorageConnectionString = storageConnectionString, - TaskHubName = GetTestTaskHubName(), - ExtendedSessionsEnabled = enableExtendedSessions, ExtendedSessionIdleTimeout = TimeSpan.FromSeconds(extendedSessionTimeoutInSeconds), + ExtendedSessionsEnabled = enableExtendedSessions, FetchLargeMessageDataEnabled = fetchLargeMessages, + StorageAccountClientProvider = new StorageAccountClientProvider(storageConnectionString), + TaskHubName = GetTestTaskHubName(), // Setting up a logger factory to enable the new DurableTask.Core logs // TODO: Add a logger provider so we can collect these logs in memory. diff --git a/test/DurableTask.Stress.Tests/Program.cs b/test/DurableTask.Stress.Tests/Program.cs index 4b383e681..654acc78e 100644 --- a/test/DurableTask.Stress.Tests/Program.cs +++ b/test/DurableTask.Stress.Tests/Program.cs @@ -42,7 +42,7 @@ static void Main(string[] args) { MaxConcurrentTaskActivityWorkItems = int.Parse(config.AppSettings.Settings["MaxConcurrentActivities"].Value), MaxConcurrentTaskOrchestrationWorkItems = int.Parse(config.AppSettings.Settings["MaxConcurrentOrchestrations"].Value), - StorageAccountDetails = new StorageAccountDetails { ConnectionString = connectionString }, + StorageAccountClientProvider = new StorageAccountClientProvider(connectionString), TaskHubName = config.AppSettings.Settings["TaskHubName"].Value, }; @@ -177,7 +177,7 @@ static void Main(string[] args) { MaxConcurrentTaskActivityWorkItems = int.Parse(ConfigurationManager.AppSettings["MaxConcurrentActivities"]), MaxConcurrentTaskOrchestrationWorkItems = int.Parse(ConfigurationManager.AppSettings["MaxConcurrentOrchestrations"]), - StorageAccountDetails = new StorageAccountDetails { ConnectionString = connectionString }, + StorageAccountClientProvider = new StorageAccountClientProvider(connectionString), TaskHubName = ConfigurationManager.AppSettings["TaskHubName"], }; From 60ebeda6a489f0e3a5a421317168974909eeede4 Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 23 Jan 2023 15:24:49 -0800 Subject: [PATCH 02/62] Update version of AzureStorage v2 to include version suffix (#851) --- src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 2e9e80eed..499637aac 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -31,7 +31,7 @@ $(MajorVersion).$(MinorVersion).0.0 - $(VersionPrefix) + $(VersionPrefix)-$(VersionSuffix) From 4cefd74047a9918a9788263880030a5a20f66ea5 Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Thu, 13 Apr 2023 12:11:34 -0700 Subject: [PATCH 03/62] Fix Bugs in new v12 Azure Storage (#868) Also merges in the latest changes from the main branch. --- .../MessageManagerTests.cs | 151 ++++++ .../Net/UriPathTests.cs | 38 ++ .../Storage/TableClientExtensionsTests.cs | 52 +++ .../ExceptionHandlingIntegrationTests.cs | 8 +- azure-pipelines-release.yml | 20 +- .../DurableTask.AzureServiceFabric.csproj | 2 +- ...zureStorageOrchestrationServiceSettings.cs | 12 +- .../DurableTask.AzureStorage.csproj | 22 +- .../ICustomTypeBinder.cs | 41 ++ .../MessageManager.cs | 51 +- .../Messaging/ControlQueue.cs | 2 +- src/DurableTask.AzureStorage/Net/UriPath.cs | 61 +++ .../Partitioning/AppLeaseManager.cs | 36 +- .../Storage/AzureStorageClient.cs | 16 +- src/DurableTask.AzureStorage/Storage/Blob.cs | 32 +- .../Storage/BlobContainer.cs | 6 +- src/DurableTask.AzureStorage/Storage/Queue.cs | 6 +- src/DurableTask.AzureStorage/Storage/Table.cs | 5 +- .../Storage/TableClientExtensions.cs | 49 ++ .../StorageAccountClientProvider.cs | 17 +- .../Tracking/AzureTableTrackingStore.cs | 7 +- .../TrackingServiceClientProvider.cs | 105 +++++ src/DurableTask.Core/Common/Utils.cs | 14 +- src/DurableTask.Core/DurableTask.Core.csproj | 6 +- src/DurableTask.Core/Logging/EventIds.cs | 4 + src/DurableTask.Core/Logging/LogEvents.cs | 83 ++++ src/DurableTask.Core/Logging/LogHelper.cs | 38 ++ .../Logging/StructuredEventSource.cs | 52 +++ .../TaskOrchestrationDispatcher.cs | 437 ++++++++++-------- .../DurableTask.Emulator.csproj | 2 +- ...StorageOrchestrationServiceSettingsTest.cs | 10 +- .../AzureStorageScenarioTests.cs | 91 ++++ .../DurableTask.AzureStorage.Tests.csproj | 2 +- .../DurableTask.Core.Tests.csproj | 2 +- .../DurableTask.Test.Orchestrations.csproj | 2 +- 35 files changed, 1169 insertions(+), 313 deletions(-) create mode 100644 Test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs create mode 100644 Test/DurableTask.AzureStorage.Tests/Net/UriPathTests.cs create mode 100644 Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs create mode 100644 src/DurableTask.AzureStorage/ICustomTypeBinder.cs create mode 100644 src/DurableTask.AzureStorage/Net/UriPath.cs create mode 100644 src/DurableTask.AzureStorage/Storage/TableClientExtensions.cs create mode 100644 src/DurableTask.AzureStorage/TrackingServiceClientProvider.cs diff --git a/Test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs b/Test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs new file mode 100644 index 000000000..e7400d5a6 --- /dev/null +++ b/Test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs @@ -0,0 +1,151 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Tests +{ + using System; + using System.Collections.Generic; + using DurableTask.AzureStorage.Storage; + using DurableTask.Core.History; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json; + + [TestClass] + public class MessageManagerTests + { + [DataTestMethod] + [DataRow("System.Collections.Generic.Dictionary`2[[System.String, System.Private.CoreLib],[System.String, System.Private.CoreLib]]")] + [DataRow("System.Collections.Generic.Dictionary`2[[System.String, mscorlib],[System.String, mscorlib]]")] + public void DeserializesStandardTypes(string dictionaryType) + { + // Given + var message = GetMessage(dictionaryType); + var messageManager = SetupMessageManager(new PrimitiveTypeBinder()); + + // When + var deserializedMessage = messageManager.DeserializeMessageData(message); + + // Then + Assert.IsInstanceOfType(deserializedMessage.TaskMessage.Event, typeof(ExecutionStartedEvent)); + ExecutionStartedEvent startedEvent = (ExecutionStartedEvent)deserializedMessage.TaskMessage.Event; + Assert.AreEqual("tagValue", startedEvent.Tags["tag1"]); + } + + [TestMethod] + public void FailsDeserializingUnknownTypes() + { + // Given + var message = GetMessage("RandomType"); + var messageManager = SetupMessageManager(new KnownTypeBinder()); + + // When/Then + Assert.ThrowsException(() => messageManager.DeserializeMessageData(message)); + } + + + [TestMethod] + public void DeserializesCustomTypes() + { + // Given + var message = GetMessage("KnownType"); + var messageManager = SetupMessageManager(new KnownTypeBinder()); + + // When + var deserializedMessage = messageManager.DeserializeMessageData(message); + + // Then + Assert.IsInstanceOfType(deserializedMessage.TaskMessage.Event, typeof(ExecutionStartedEvent)); + ExecutionStartedEvent startedEvent = (ExecutionStartedEvent)deserializedMessage.TaskMessage.Event; + Assert.AreEqual("tagValue", startedEvent.Tags["tag1"]); + } + + [DataTestMethod] + [DataRow("blob.bin")] + [DataRow("@#$%!")] + [DataRow("foo/bar/b@z.tar.gz")] + public void GetBlobUrlUnescaped(string blob) + { + var settings = new AzureStorageOrchestrationServiceSettings + { + StorageAccountClientProvider = new StorageAccountClientProvider("UseDevelopmentStorage=true"), + }; + + const string container = "@entity12345"; + var manager = new MessageManager(settings, new AzureStorageClient(settings), container); + var expected = $"http://127.0.0.1:10000/devstoreaccount1/{container}/{blob}"; + Assert.AreEqual(expected, manager.GetBlobUrl(blob)); + } + + private string GetMessage(string dictionaryType) + => "{\"$type\":\"DurableTask.AzureStorage.MessageData\",\"ActivityId\":\"5406d369-4369-4673-afae-6671a2fa1e57\",\"TaskMessage\":{\"$type\":\"DurableTask.Core.TaskMessage\",\"Event\":{\"$type\":\"DurableTask.Core.History.ExecutionStartedEvent\",\"OrchestrationInstance\":{\"$type\":\"DurableTask.Core.OrchestrationInstance\",\"InstanceId\":\"2.2-34a2c9d4-306e-4467-8470-a8018b2e4f11\",\"ExecutionId\":\"aae324dcc8f943e490b37ec5e5bbf9da\"},\"EventType\":0,\"ParentInstance\":null,\"Name\":\"OrchestrationName\",\"Version\":\"2.0\",\"Input\":\"input\",\"Tags\":{\"$type\":\"" + + dictionaryType + + "\",\"tag1\":\"tagValue\"},\"Correlation\":null,\"ScheduledStartTime\":null,\"Generation\":0,\"EventId\":-1,\"IsPlayed\":false,\"Timestamp\":\"2023-03-24T20:53:05.9093518Z\"},\"SequenceNumber\":0,\"OrchestrationInstance\":{\"$type\":\"DurableTask.Core.OrchestrationInstance\",\"InstanceId\":\"2.2-34a2c9d4-306e-4467-8470-a8018b2e4f11\",\"ExecutionId\":\"aae324dcc8f943e490b37ec5e5bbf9da\"}},\"CompressedBlobName\":null,\"SequenceNumber\":40,\"Sender\":{\"InstanceId\":\"\",\"ExecutionId\":\"\"},\"SerializableTraceContext\":null}\r\n\r\n"; + + private MessageManager SetupMessageManager(ICustomTypeBinder binder) + { + var azureStorageClient = new AzureStorageClient( + new AzureStorageOrchestrationServiceSettings + { + StorageAccountClientProvider = new StorageAccountClientProvider("UseDevelopmentStorage=true"), + }); + + return new MessageManager( + new AzureStorageOrchestrationServiceSettings { CustomMessageTypeBinder = binder }, + azureStorageClient, + "$root"); + } + } + + internal class KnownTypeBinder : ICustomTypeBinder + { + public void BindToName(Type serializedType, out string assemblyName, out string typeName) + { + throw new NotImplementedException(); + } + + public Type? BindToType(string assemblyName, string typeName) + { + if (typeName == "KnownType") + { + return typeof(Dictionary); + } + + return null; + } + } + + internal class PrimitiveTypeBinder : ICustomTypeBinder + { + readonly bool hasStandardLib; + + public PrimitiveTypeBinder() + { + hasStandardLib = typeof(string).AssemblyQualifiedName!.Contains("mscorlib"); + } + + public void BindToName(Type serializedType, out string assemblyName, out string typeName) + { + throw new NotImplementedException(); + } + + public Type BindToType(string assemblyName, string typeName) + { + if (hasStandardLib) + { + return Type.GetType(typeName.Replace("System.Private.CoreLib", "mscorlib"))!; + } + + return Type.GetType(typeName.Replace("mscorlib", "System.Private.CoreLib"))!; + } + } +} diff --git a/Test/DurableTask.AzureStorage.Tests/Net/UriPathTests.cs b/Test/DurableTask.AzureStorage.Tests/Net/UriPathTests.cs new file mode 100644 index 000000000..78703d5fd --- /dev/null +++ b/Test/DurableTask.AzureStorage.Tests/Net/UriPathTests.cs @@ -0,0 +1,38 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Net +{ + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class UriPathTests + { + [DataTestMethod] + [DataRow("", "", "")] + [DataRow("", "bar/baz", "bar/baz")] + [DataRow("foo", "", "foo")] + [DataRow("foo", "/", "foo/")] + [DataRow("foo", "bar", "foo/bar")] + [DataRow("foo", "/bar", "foo/bar")] + [DataRow("foo/", "", "foo/")] + [DataRow("foo/", "/", "foo/")] + [DataRow("foo/", "bar", "foo/bar")] + [DataRow("foo/", "/bar", "foo/bar")] + [DataRow("/foo//", "//bar/baz", "/foo///bar/baz")] + public void Combine(string path1, string path2, string expected) + { + Assert.AreEqual(expected, UriPath.Combine(path1, path2)); + } + } +} diff --git a/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs b/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs new file mode 100644 index 000000000..430d91ce3 --- /dev/null +++ b/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs @@ -0,0 +1,52 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.AzureStorage.Tests.Storage +{ + using System; + using Azure.Data.Tables; + using DurableTask.AzureStorage.Storage; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class TableClientExtensionsTests + { + const string EmulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + + [TestMethod] + public void GetUri_ConnectionString() + { + TableClient client; + + client = new TableClient("UseDevelopmentStorage=true", "bar"); + Assert.AreEqual(new Uri("http://127.0.0.1:10002/devstoreaccount1/bar"), client.GetUri()); + + client = new TableClient($"DefaultEndpointsProtocol=https;AccountName=foo;AccountKey={EmulatorAccountKey};TableEndpoint=https://foo.table.core.windows.net/;", "bar"); + Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri()); + } + + [TestMethod] + public void GetUri_ServiceEndpoint() + { + var client = new TableClient(new Uri("https://foo.table.core.windows.net/"), "bar", new TableSharedKeyCredential("foo", EmulatorAccountKey)); + Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri()); + } + + [TestMethod] + public void GetUri_TableEndpoint() + { + var client = new TableClient(new Uri("https://foo.table.core.windows.net/bar")); + Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri()); + } + } +} \ No newline at end of file diff --git a/Test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs b/Test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs index 6c2e051d1..9e8042254 100644 --- a/Test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs +++ b/Test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs @@ -61,16 +61,16 @@ await this.worker if (mode == ErrorPropagationMode.SerializeExceptions) { // The exception should be deserializable - InvalidOperationException e = JsonConvert.DeserializeObject(state.Output); + InvalidOperationException? e = JsonConvert.DeserializeObject(state.Output); Assert.IsNotNull(e); - Assert.AreEqual("This is a test exception", e.Message); + Assert.AreEqual("This is a test exception", e!.Message); } else if (mode == ErrorPropagationMode.UseFailureDetails) { // The failure details should contain the relevant exception metadata - FailureDetails details = JsonConvert.DeserializeObject(state.Output); + FailureDetails? details = JsonConvert.DeserializeObject(state.Output); Assert.IsNotNull(details); - Assert.AreEqual(typeof(InvalidOperationException).FullName, details.ErrorType); + Assert.AreEqual(typeof(InvalidOperationException).FullName, details!.ErrorType); Assert.IsTrue(details.IsCausedBy()); Assert.IsTrue(details.IsCausedBy()); // check that base types work too Assert.AreEqual("This is a test exception", details.ErrorMessage); diff --git a/azure-pipelines-release.yml b/azure-pipelines-release.yml index f99d06bb1..b0a9559b6 100644 --- a/azure-pipelines-release.yml +++ b/azure-pipelines-release.yml @@ -138,16 +138,16 @@ steps: packDirectory: $(build.artifactStagingDirectory) packagesToPack: 'src/DurableTask.ServiceBus/DurableTask.ServiceBus.csproj' -# - task: DotNetCoreCLI@2 -# displayName: Generate nuget packages -# inputs: -# command: pack -# verbosityPack: Minimal -# configuration: Release -# nobuild: true -# packDirectory: $(build.artifactStagingDirectory) -# packagesToPack: 'src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj' -# buildProperties: 'Platform=x64' +- task: DotNetCoreCLI@2 + displayName: Generate nuget packages + inputs: + command: pack + verbosityPack: Minimal + configuration: Release + nobuild: true + packDirectory: $(build.artifactStagingDirectory) + packagesToPack: 'src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj' + buildProperties: 'Platform=x64' # Digitally sign all the nuget packages with the Microsoft certificate. # This appears to be an in-place signing job, which is convenient. diff --git a/src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj b/src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj index 45c3a5725..619395503 100644 --- a/src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj +++ b/src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj @@ -6,7 +6,7 @@ true Microsoft.Azure.DurableTask.AzureServiceFabric true - 2.3.6 + 2.3.7 $(Version) $(Version) Azure Service Fabric provider extension for the Durable Task Framework. diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs index e29a3f0da..90a76bd1c 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs @@ -150,7 +150,7 @@ public class AzureStorageOrchestrationServiceSettings public bool UseAppLease { get; set; } = true; /// - /// If UseAppLease is true, gets or sets the AppLeaaseOptions used for acquiring the lease to start the application. + /// If UseAppLease is true, gets or sets the AppLeaseOptions used for acquiring the lease to start the application. /// public AppLeaseOptions AppLeaseOptions { get; set; } = AppLeaseOptions.DefaultOptions; @@ -160,13 +160,12 @@ public class AzureStorageOrchestrationServiceSettings public StorageAccountClientProvider? StorageAccountClientProvider { get; set; } /// - /// Gets or sets the storage provider for Azure Table Storage, which is used for the Tracking Store - /// that records the progress of orchestrations and entities. + /// Gets or sets the optional client provider for the Tracking Store that records the progress of orchestrations and entities. /// /// /// If the value is , then the is used instead. /// - public IStorageServiceClientProvider? TrackingServiceClientProvider { get; set; } + public TrackingServiceClientProvider? TrackingServiceClientProvider { get; set; } /// /// Should we carry over unexecuted raised events to the next iteration of an orchestration on ContinueAsNew @@ -199,6 +198,11 @@ public class AzureStorageOrchestrationServiceSettings /// public bool DisableExecutionStartedDeduplication { get; set; } + /// + /// Gets or sets an optional custom type binder used when trying to deserialize queued messages. + /// + public ICustomTypeBinder? CustomMessageTypeBinder { get; set; } + /// /// Returns bool indicating is the has been set. /// diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 499637aac..7acdc2ba7 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -8,11 +8,11 @@ Azure Storage provider extension for the Durable Task Framework. Azure Task Durable Orchestration Workflow Activity Reliable AzureStorage Microsoft.Azure.DurableTask.AzureStorage - true - true - true - embedded - false + true + true + true + embedded + false NU5125;NU5048 @@ -24,7 +24,7 @@ 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) - preview.1 + preview.2 $(VersionPrefix).0 $(VersionPrefix).$(FileVersionRevision) @@ -34,11 +34,14 @@ $(VersionPrefix)-$(VersionSuffix) + - - - + + + + + @@ -51,7 +54,6 @@ - diff --git a/src/DurableTask.AzureStorage/ICustomTypeBinder.cs b/src/DurableTask.AzureStorage/ICustomTypeBinder.cs new file mode 100644 index 000000000..44e85fdd3 --- /dev/null +++ b/src/DurableTask.AzureStorage/ICustomTypeBinder.cs @@ -0,0 +1,41 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace DurableTask.AzureStorage +{ + /// + /// Abstraction to prevent surfacing the dynamic loading between System.Runtime.Serialization.SerializationBinder and Newtonsoft.Json.Serialization.ISerializationBinder + /// Used when deserializing QueueMessages to MessageData to allow providing custom type binding. + /// Does not support custom bindings for DurableTask types + /// + public interface ICustomTypeBinder + { + /// + /// When implemented, controls the binding of a serialized object to a type. + /// + /// The type of the object the formatter creates a new instance of. + /// Specifies the System.Reflection.Assembly name of the serialized object. + /// Specifies the System.Type name of the serialized object. + void BindToName(Type serializedType, out string assemblyName, out string typeName); + + /// + /// When implemented, controls the binding of a serialized object to a type. + /// + /// Specifies the System.Reflection.Assembly name of the serialized object. + /// Specifies the System.Type name of the serialized object + /// The type of the object the formatter creates a new instance of. + Type BindToType(string assemblyName, string typeName); + } +} \ No newline at end of file diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index c97bc26a9..f1e543783 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -62,9 +62,9 @@ class MessageManager { TypeNameHandling = TypeNameHandling.Objects, #if NETSTANDARD2_0 - SerializationBinder = new TypeNameSerializationBinder(), + SerializationBinder = new TypeNameSerializationBinder(settings.CustomMessageTypeBinder), #else - Binder = new TypeNameSerializationBinder(), + Binder = new TypeNameSerializationBinder(settings.CustomMessageTypeBinder), #endif }; this.serializer = JsonSerializer.Create(taskMessageSerializerSettings); @@ -138,7 +138,13 @@ public async Task FetchLargeMessageIfNecessary(string message, Cancellat internal static bool TryGetLargeMessageReference(string messagePayload, out Uri blobUrl) { - return Uri.TryCreate(messagePayload, UriKind.Absolute, out blobUrl); + if (Uri.IsWellFormedUriString(messagePayload, UriKind.Absolute)) + { + return Uri.TryCreate(messagePayload, UriKind.Absolute, out blobUrl); + } + + blobUrl = null; + return false; } public async Task DeserializeQueueMessageAsync(QueueMessage queueMessage, string queueName, CancellationToken cancellationToken = default) @@ -233,7 +239,7 @@ private async Task DownloadAndDecompressAsBytesAsync(Blob blob, Cancella public string GetBlobUrl(string blobName) { - return this.blobContainer.GetBlobReference(blobName).AbsoluteUri; + return Uri.UnescapeDataString(this.blobContainer.GetBlobReference(blobName).Uri.AbsoluteUri); } public ArraySegment Decompress(Stream blobStream) @@ -312,27 +318,39 @@ await foreach (Page page in this.blobContainer.ListBlobsAsync(sanitizedIns #if NETSTANDARD2_0 class TypeNameSerializationBinder : ISerializationBinder { + readonly ICustomTypeBinder customBinder; + public TypeNameSerializationBinder(ICustomTypeBinder customBinder) + { + this.customBinder = customBinder; + } + public void BindToName(Type serializedType, out string assemblyName, out string typeName) { - TypeNameSerializationHelper.BindToName(serializedType, out assemblyName, out typeName); + TypeNameSerializationHelper.BindToName(customBinder, serializedType, out assemblyName, out typeName); } public Type BindToType(string assemblyName, string typeName) { - return TypeNameSerializationHelper.BindToType(assemblyName, typeName); + return TypeNameSerializationHelper.BindToType(customBinder, assemblyName, typeName); } } #else class TypeNameSerializationBinder : SerializationBinder { + readonly ICustomTypeBinder customBinder; + public TypeNameSerializationBinder(ICustomTypeBinder customBinder) + { + this.customBinder = customBinder; + } + public override void BindToName(Type serializedType, out string assemblyName, out string typeName) { - TypeNameSerializationHelper.BindToName(serializedType, out assemblyName, out typeName); + TypeNameSerializationHelper.BindToName(customBinder, serializedType, out assemblyName, out typeName); } public override Type BindToType(string assemblyName, string typeName) { - return TypeNameSerializationHelper.BindToType(assemblyName, typeName); + return TypeNameSerializationHelper.BindToType(customBinder, assemblyName, typeName); } } #endif @@ -341,13 +359,20 @@ static class TypeNameSerializationHelper static readonly Assembly DurableTaskCore = typeof(DurableTask.Core.TaskMessage).Assembly; static readonly Assembly DurableTaskAzureStorage = typeof(AzureStorageOrchestrationService).Assembly; - public static void BindToName(Type serializedType, out string assemblyName, out string typeName) + public static void BindToName(ICustomTypeBinder customBinder, Type serializedType, out string assemblyName, out string typeName) { - assemblyName = null; - typeName = serializedType.FullName; + if (customBinder != null) + { + customBinder.BindToName(serializedType, out assemblyName, out typeName); + } + else + { + assemblyName = null; + typeName = serializedType.FullName; + } } - public static Type BindToType(string assemblyName, string typeName) + public static Type BindToType(ICustomTypeBinder customBinder, string assemblyName, string typeName) { if (typeName.StartsWith("DurableTask.Core")) { @@ -358,7 +383,7 @@ public static Type BindToType(string assemblyName, string typeName) return DurableTaskAzureStorage.GetType(typeName, throwOnError: true); } - return Type.GetType(typeName, throwOnError: true); + return customBinder?.BindToType(assemblyName, typeName) ?? Type.GetType(typeName, throwOnError: true); } } } diff --git a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs index 68e924a50..faad07453 100644 --- a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs +++ b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs @@ -185,7 +185,7 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) } } - this.Release(CloseReason.Shutdown, "ControlQueue GetMessagesAsync"); + this.Release(CloseReason.Shutdown, $"ControlQueue GetMessagesAsync cancelled by: {(this.releaseCancellationToken.IsCancellationRequested ? "control queue released token cancelled" : "")} {(cancellationToken.IsCancellationRequested ? "shutdown token cancelled" : "")}"); return EmptyMessageList; } } diff --git a/src/DurableTask.AzureStorage/Net/UriPath.cs b/src/DurableTask.AzureStorage/Net/UriPath.cs new file mode 100644 index 000000000..6083b787f --- /dev/null +++ b/src/DurableTask.AzureStorage/Net/UriPath.cs @@ -0,0 +1,61 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Net +{ + using System; + + static class UriPath + { + public static string Combine(string path1, string path2) + { + if (path1 == null) + { + throw new ArgumentNullException(nameof(path1)); + } + + if (path2 == null) + { + throw new ArgumentNullException(nameof(path2)); + } + + if (path1.Length == 0) + { + return path2; + } + else if (path2.Length == 0) + { + return path1; + } + + // Path1 ends with a '/' + // E.g. path1 = "foo/" + if (path1[path1.Length - 1] == '/') + { + return path2[0] == '/' + ? path1 + path2.Substring(1) // E.g. Combine("foo/", "/bar") == "foo/bar" + : path1 + path2; // E.g. Combine("foo/", "bar") == "foo/bar" + } + else if (path2[0] == '/') + { + // E.g. Combine("foo", "/bar") == "foo/bar" + return path1 + path2; + } + else + { + // E.g. Combine("foo", "bar") == "foo/bar" + return path1 + '/' + path2; + } + } + } +} diff --git a/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs b/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs index 35550e1e3..9be4dc366 100644 --- a/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs +++ b/src/DurableTask.AzureStorage/Partitioning/AppLeaseManager.cs @@ -46,6 +46,7 @@ sealed class AppLeaseManager bool isLeaseOwner; int appLeaseIsStarted; Task renewTask; + Task acquireTask; CancellationTokenSource starterTokenSource; CancellationTokenSource leaseRenewerCancellationTokenSource; @@ -121,9 +122,14 @@ async Task RestartAppLeaseStarterTask() this.starterTokenSource.Cancel(); this.starterTokenSource.Dispose(); } - this.starterTokenSource = new CancellationTokenSource(); - await Task.Factory.StartNew(() => this.AppLeaseManagerStarter(this.starterTokenSource.Token)); + if (this.acquireTask != null) + { + await this.acquireTask; + } + + this.starterTokenSource = new CancellationTokenSource(); + this.acquireTask = await Task.Factory.StartNew(() => this.AppLeaseManagerStarter(this.starterTokenSource.Token)); } async Task AppLeaseManagerStarter(CancellationToken cancellationToken) @@ -132,7 +138,7 @@ async Task AppLeaseManagerStarter(CancellationToken cancellationToken) { try { - while (!await this.TryAquireAppLeaseAsync()) + while (!await this.TryAcquireAppLeaseAsync()) { await Task.Delay(this.settings.AppLeaseOptions.AcquireInterval, cancellationToken); } @@ -159,6 +165,18 @@ async Task AppLeaseManagerStarter(CancellationToken cancellationToken) public async Task StopAsync() { + if (this.starterTokenSource != null) + { + this.starterTokenSource.Cancel(); + this.starterTokenSource.Dispose(); + this.starterTokenSource = null; + } + + if (this.acquireTask != null) + { + await this.acquireTask; + } + if (this.appLeaseIsEnabled) { await this.StopAppLeaseAsync(); @@ -168,12 +186,6 @@ public async Task StopAsync() await this.partitionManager.StopAsync(); } - if (this.starterTokenSource != null) - { - this.starterTokenSource.Cancel(); - this.starterTokenSource.Dispose(); - this.starterTokenSource = null; - } } public async Task ForceChangeAppLeaseAsync() @@ -270,7 +282,7 @@ async Task StopAppLeaseAsync() this.leaseRenewerCancellationTokenSource?.Dispose(); } - async Task TryAquireAppLeaseAsync() + async Task TryAcquireAppLeaseAsync() { AppLeaseInfo appLeaseInfo = await this.GetAppLeaseInfoAsync(); @@ -281,7 +293,7 @@ async Task TryAquireAppLeaseAsync() } else { - leaseAcquired = await this.TryAquireLeaseAsync(); + leaseAcquired = await this.TryAcquireLeaseAsync(); } this.isLeaseOwner = leaseAcquired; @@ -349,7 +361,7 @@ async Task ChangeLeaseAsync(string currentLeaseId) return leaseAcquired; } - async Task TryAquireLeaseAsync() + async Task TryAcquireLeaseAsync() { bool leaseAcquired; diff --git a/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs b/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs index f4d8468cc..57a134bba 100644 --- a/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs +++ b/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs @@ -46,13 +46,17 @@ public AzureStorageClient(AzureStorageOrchestrationServiceSettings settings) var timeoutPolicy = new LeaseTimeoutHttpPipelinePolicy(this.Settings.LeaseRenewInterval); var monitoringPolicy = new MonitoringHttpPipelinePolicy(this.Stats); - this.blobClient = CreateClient(settings.StorageAccountClientProvider.Blob, ConfigureClientPolicies); this.queueClient = CreateClient(settings.StorageAccountClientProvider.Queue, ConfigureClientPolicies); - this.tableClient = CreateClient( - settings.HasTrackingStoreStorageAccount - ? settings.TrackingServiceClientProvider! - : settings.StorageAccountClientProvider.Table, - ConfigureClientPolicies); + if (settings.HasTrackingStoreStorageAccount) + { + this.blobClient = CreateClient(settings.TrackingServiceClientProvider!.Blob, ConfigureClientPolicies); + this.tableClient = CreateClient(settings.TrackingServiceClientProvider!.Table, ConfigureClientPolicies); + } + else + { + this.blobClient = CreateClient(settings.StorageAccountClientProvider.Blob, ConfigureClientPolicies); + this.tableClient = CreateClient(settings.StorageAccountClientProvider.Table, ConfigureClientPolicies); + } void ConfigureClientPolicies(TClientOptions options) where TClientOptions : ClientOptions { diff --git a/src/DurableTask.AzureStorage/Storage/Blob.cs b/src/DurableTask.AzureStorage/Storage/Blob.cs index 03f664fb0..22b92116f 100644 --- a/src/DurableTask.AzureStorage/Storage/Blob.cs +++ b/src/DurableTask.AzureStorage/Storage/Blob.cs @@ -30,41 +30,27 @@ class Blob public Blob(BlobServiceClient blobServiceClient, string containerName, string blobName) { - this.Name = blobName; - this.blockBlobClient = blobServiceClient.GetBlobContainerClient(containerName).GetBlockBlobClient(blobName); + this.blockBlobClient = blobServiceClient + .GetBlobContainerClient(containerName) + .GetBlockBlobClient(blobName); } public Blob(BlobServiceClient blobServiceClient, Uri blobUri) { - string serviceString = blobServiceClient.Uri.AbsoluteUri; - if (serviceString[serviceString.Length - 1] != '/') - { - serviceString += '/'; - } - - string blobString = blobUri.AbsoluteUri; - if (blobString.IndexOf(serviceString, StringComparison.Ordinal) != 0) + if (!blobUri.AbsoluteUri.StartsWith(blobServiceClient.Uri.AbsoluteUri, StringComparison.Ordinal)) { throw new ArgumentException("Blob is not present in the storage account", nameof(blobUri)); } - // Create a relative URI by removing the service's address - // ie. // -> / - string remaining = blobString.Substring(serviceString.Length); - int containerEnd = remaining.IndexOf('/'); - if (containerEnd == -1) - { - throw new ArgumentException("Missing blob container", nameof(blobUri)); - } - + var builder = new BlobUriBuilder(blobUri); this.blockBlobClient = blobServiceClient - .GetBlobContainerClient(remaining.Substring(0, containerEnd)) - .GetBlockBlobClient(remaining.Substring(containerEnd + 1)); + .GetBlobContainerClient(builder.BlobContainerName) + .GetBlockBlobClient(builder.BlobName); } - public string? Name { get; } + public string Name => this.blockBlobClient.Name; - public string AbsoluteUri => this.blockBlobClient.Uri.AbsoluteUri; + public Uri Uri => this.blockBlobClient.Uri; public async Task ExistsAsync(CancellationToken cancellationToken = default) { diff --git a/src/DurableTask.AzureStorage/Storage/BlobContainer.cs b/src/DurableTask.AzureStorage/Storage/BlobContainer.cs index 175624db1..120394151 100644 --- a/src/DurableTask.AzureStorage/Storage/BlobContainer.cs +++ b/src/DurableTask.AzureStorage/Storage/BlobContainer.cs @@ -14,15 +14,13 @@ namespace DurableTask.AzureStorage.Storage { using System; - using System.Collections.Generic; - using System.IO; - using System.Linq; using System.Threading; using System.Threading.Tasks; using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; + using DurableTask.AzureStorage.Net; class BlobContainer { @@ -40,7 +38,7 @@ public BlobContainer(AzureStorageClient azureStorageClient, BlobServiceClient bl public Blob GetBlobReference(string blobName, string? blobPrefix = null) { - string fullBlobName = blobPrefix != null ? Path.Combine(blobPrefix, blobName) : blobName; + string fullBlobName = blobPrefix != null ? UriPath.Combine(blobPrefix, blobName) : blobName; return this.azureStorageClient.GetBlobReference(this.containerName, fullBlobName); } diff --git a/src/DurableTask.AzureStorage/Storage/Queue.cs b/src/DurableTask.AzureStorage/Storage/Queue.cs index e82f588ea..46b4115d8 100644 --- a/src/DurableTask.AzureStorage/Storage/Queue.cs +++ b/src/DurableTask.AzureStorage/Storage/Queue.cs @@ -32,12 +32,10 @@ public Queue(AzureStorageClient azureStorageClient, QueueServiceClient queueServ { this.azureStorageClient = azureStorageClient; this.stats = this.azureStorageClient.Stats; - this.Name = queueName; - - this.queueClient = queueServiceClient.GetQueueClient(this.Name); + this.queueClient = queueServiceClient.GetQueueClient(queueName); } - public string Name { get; } + public string Name => this.queueClient.Name; public Uri Uri => this.queueClient.Uri; diff --git a/src/DurableTask.AzureStorage/Storage/Table.cs b/src/DurableTask.AzureStorage/Storage/Table.cs index 8cca7dc08..833a3778c 100644 --- a/src/DurableTask.AzureStorage/Storage/Table.cs +++ b/src/DurableTask.AzureStorage/Storage/Table.cs @@ -34,16 +34,15 @@ class Table public Table(AzureStorageClient azureStorageClient, TableServiceClient tableServiceClient, string tableName) { this.azureStorageClient = azureStorageClient; - this.Name = tableName; this.stats = this.azureStorageClient.Stats; this.tableServiceClient = tableServiceClient; this.tableClient = tableServiceClient.GetTableClient(tableName); } - public string Name { get; } + public string Name => this.tableClient.Name; - internal Uri Uri => this.tableClient.Uri; + internal Uri? Uri => this.tableClient.GetUri(); // TODO: Replace with Uri property public async Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default) { diff --git a/src/DurableTask.AzureStorage/Storage/TableClientExtensions.cs b/src/DurableTask.AzureStorage/Storage/TableClientExtensions.cs new file mode 100644 index 000000000..d3a77d17b --- /dev/null +++ b/src/DurableTask.AzureStorage/Storage/TableClientExtensions.cs @@ -0,0 +1,49 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Storage +{ + using System; + using System.Linq.Expressions; + using System.Reflection; + using Azure.Data.Tables; + + static class TableClientExtensions + { + public static Uri? GetUri(this TableClient tableClient) + { + if (tableClient == null) + { + throw new ArgumentNullException(nameof(tableClient)); + } + + Uri? endpoint = GetEndpointFunc(tableClient); + return endpoint != null ? new TableUriBuilder(endpoint) { Tablename = tableClient.Name }.ToUri() : null; + } + + static readonly Func GetEndpointFunc = CreateGetEndpointFunc(); + + static Func CreateGetEndpointFunc() + { + Type tableClientType = typeof(TableClient); + ParameterExpression clientParam = Expression.Parameter(typeof(TableClient), "client"); + FieldInfo? endpointField = tableClientType.GetField("_endpoint", BindingFlags.Instance | BindingFlags.NonPublic); + + Expression> lambdaExpr = endpointField != null + ? Expression.Lambda>(Expression.Field(clientParam, endpointField), clientParam) + : Expression.Lambda>(Expression.Constant(null, typeof(Uri)), clientParam); + + return lambdaExpr.Compile(); + } + } +} diff --git a/src/DurableTask.AzureStorage/StorageAccountClientProvider.cs b/src/DurableTask.AzureStorage/StorageAccountClientProvider.cs index cb7a38baf..76cecb1bc 100644 --- a/src/DurableTask.AzureStorage/StorageAccountClientProvider.cs +++ b/src/DurableTask.AzureStorage/StorageAccountClientProvider.cs @@ -23,7 +23,7 @@ namespace DurableTask.AzureStorage /// /// Represents a client provider for the services exposed by an Azure Storage Account. /// - public sealed class StorageAccountClientProvider + public sealed class StorageAccountClientProvider : TrackingServiceClientProvider { /// /// Initializes a new instance of the class that returns @@ -68,7 +68,7 @@ public StorageAccountClientProvider(string accountName, TokenCredential tokenCre /// An Azure Blob Storage service URI. /// An Azure Queue Storage service URI. /// An Azure Table Storage service URI. - /// A token credential for accessing the service. + /// A token credential for accessing the storage services. /// /// , , /// , or is . @@ -94,25 +94,14 @@ public StorageAccountClientProvider(Uri blobServiceUri, Uri queueServiceUri, Uri IStorageServiceClientProvider blob, IStorageServiceClientProvider queue, IStorageServiceClientProvider table) + : base(blob, table) { - this.Blob = blob ?? throw new ArgumentNullException(nameof(blob)); this.Queue = queue ?? throw new ArgumentNullException(nameof(queue)); - this.Table = table ?? throw new ArgumentNullException(nameof(table)); } - /// - /// Gets the client provider for Azure Blob Storage. - /// - public IStorageServiceClientProvider Blob { get; } - /// /// Gets the client provider for Azure Queue Storage. /// public IStorageServiceClientProvider Queue { get; } - - /// - /// Gets the client provider for Azure Table Storage. - /// - public IStorageServiceClientProvider Table { get; } } } diff --git a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs index f607c2c67..af078f50b 100644 --- a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs @@ -787,8 +787,11 @@ public override async Task UpdateStatusForRewindAsync(string instanceId, Cancell /// public override Task StartAsync(CancellationToken cancellationToken = default) { - ServicePointManager.FindServicePoint(this.HistoryTable.Uri).UseNagleAlgorithm = false; - ServicePointManager.FindServicePoint(this.InstancesTable.Uri).UseNagleAlgorithm = false; + if (this.HistoryTable.Uri != null) + ServicePointManager.FindServicePoint(this.HistoryTable.Uri).UseNagleAlgorithm = false; + + if (this.InstancesTable.Uri != null) + ServicePointManager.FindServicePoint(this.InstancesTable.Uri).UseNagleAlgorithm = false; return Task.CompletedTask; } diff --git a/src/DurableTask.AzureStorage/TrackingServiceClientProvider.cs b/src/DurableTask.AzureStorage/TrackingServiceClientProvider.cs new file mode 100644 index 000000000..8e0856702 --- /dev/null +++ b/src/DurableTask.AzureStorage/TrackingServiceClientProvider.cs @@ -0,0 +1,105 @@ + +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage +{ + using System; + using Azure.Core; + using Azure.Data.Tables; + using Azure.Storage.Blobs; + + /// + /// Represents a client provider for the services used to track the execution of Durable Tasks. + /// + public class TrackingServiceClientProvider + { + /// + /// Initializes a new instance of the class that returns + /// service clients using the given . + /// + /// An Azure Storage connection string. + /// + /// is or consists entirely of white space characters. + /// + public TrackingServiceClientProvider(string connectionString) + : this( + StorageServiceClientProvider.ForBlob(connectionString), + StorageServiceClientProvider.ForTable(connectionString)) + { } + + /// + /// Initializes a new instance of the class that returns + /// service clients using the given and credential. + /// + /// An Azure Storage account name. + /// A token credential for accessing the service. + /// An Azure Blob Storage service client whose connection is based on the given . + /// + /// + /// is or consists entirely of white space characters. + /// + /// -or- + /// is . + /// + public TrackingServiceClientProvider(string accountName, TokenCredential tokenCredential) + : this( + StorageServiceClientProvider.ForBlob(accountName, tokenCredential), + StorageServiceClientProvider.ForTable(accountName, tokenCredential)) + { } + + /// + /// Initializes a new instance of the class that returns + /// service clients using the given service URIs and credential. + /// + /// An Azure Blob Storage service URI. + /// An Azure Table Storage service URI. + /// A token credential for accessing the storage services. + /// + /// , , + /// or is . + /// + public TrackingServiceClientProvider(Uri blobServiceUri, Uri tableServiceUri, TokenCredential tokenCredential) + : this( + StorageServiceClientProvider.ForBlob(blobServiceUri, tokenCredential), + StorageServiceClientProvider.ForTable(tableServiceUri, tokenCredential)) + { } + + /// + /// Initializes a new instance of the class that returns + /// service clients using the given client providers. + /// + /// An Azure Blob Storage service client provider. + /// An Azure Table Storage service client provider. + /// + /// or is . + /// + public TrackingServiceClientProvider( + IStorageServiceClientProvider blob, + IStorageServiceClientProvider table) + { + this.Blob = blob ?? throw new ArgumentNullException(nameof(blob)); + this.Table = table ?? throw new ArgumentNullException(nameof(table)); + } + + /// + /// Gets the client provider for Azure Blob Storage. + /// + public IStorageServiceClientProvider Blob { get; } + + /// + /// Gets the client provider for Azure Table Storage. + /// + public IStorageServiceClientProvider Table { get; } + } +} diff --git a/src/DurableTask.Core/Common/Utils.cs b/src/DurableTask.Core/Common/Utils.cs index 2d63eb895..32b5939dd 100644 --- a/src/DurableTask.Core/Common/Utils.cs +++ b/src/DurableTask.Core/Common/Utils.cs @@ -102,9 +102,9 @@ public static string SerializeToJson(JsonSerializer serializer, object payload) /// The serializer whose config will guide the deserialization. /// The JSON-string to deserialize. /// - public static T DeserializeFromJson(JsonSerializer serializer, string jsonString) + public static T? DeserializeFromJson(JsonSerializer serializer, string jsonString) { - T obj; + T? obj; using (var reader = new StringReader(jsonString)) using (var jsonReader = new JsonTextReader(reader)) { @@ -120,7 +120,7 @@ public static T DeserializeFromJson(JsonSerializer serializer, string jsonStr /// The JSON-string to deserialize. /// The expected de-serialization type. /// - public static object DeserializeFromJson(string jsonString, Type type) + public static object? DeserializeFromJson(string jsonString, Type type) { return DeserializeFromJson(DefaultSerializer, jsonString, type); } @@ -133,9 +133,9 @@ public static object DeserializeFromJson(string jsonString, Type type) /// The JSON-string to deserialize. /// The expected de-serialization type. /// - public static object DeserializeFromJson(JsonSerializer serializer, string jsonString, Type type) + public static object? DeserializeFromJson(JsonSerializer serializer, string jsonString, Type type) { - object obj; + object? obj; using (var reader = new StringReader(jsonString)) using (var jsonReader = new JsonTextReader(reader)) { @@ -258,7 +258,7 @@ public static Stream WriteStringToStream(string input, bool compress, out long o /// /// Reads and deserializes an Object from the supplied stream /// - public static T ReadObjectFromStream(Stream objectStream) + public static T? ReadObjectFromStream(Stream objectStream) { return ReadObjectFromByteArray(ReadBytesFromStream(objectStream)); } @@ -285,7 +285,7 @@ public static byte[] ReadBytesFromStream(Stream objectStream) /// /// Deserializes an Object from the supplied bytes /// - public static T ReadObjectFromByteArray(byte[] serializedBytes) + public static T? ReadObjectFromByteArray(byte[] serializedBytes) { var jsonString = Encoding.UTF8.GetString(serializedBytes); return DeserializeFromJson(DefaultObjectJsonSerializer, jsonString); diff --git a/src/DurableTask.Core/DurableTask.Core.csproj b/src/DurableTask.Core/DurableTask.Core.csproj index cb8c916f2..8c1983c72 100644 --- a/src/DurableTask.Core/DurableTask.Core.csproj +++ b/src/DurableTask.Core/DurableTask.Core.csproj @@ -17,8 +17,8 @@ 2 - 12 - 1 + 13 + 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) $(VersionPrefix).0 @@ -37,7 +37,7 @@ - + diff --git a/src/DurableTask.Core/Logging/EventIds.cs b/src/DurableTask.Core/Logging/EventIds.cs index af20b11a8..963de5f71 100644 --- a/src/DurableTask.Core/Logging/EventIds.cs +++ b/src/DurableTask.Core/Logging/EventIds.cs @@ -59,5 +59,9 @@ static class EventIds public const int SuspendingInstance = 68; public const int ResumingInstance = 69; + + public const int RenewOrchestrationWorkItemStarting = 70; + public const int RenewOrchestrationWorkItemCompleted = 71; + public const int RenewOrchestrationWorkItemFailed = 72; } } diff --git a/src/DurableTask.Core/Logging/LogEvents.cs b/src/DurableTask.Core/Logging/LogEvents.cs index d34cc5fcb..a914154a2 100644 --- a/src/DurableTask.Core/Logging/LogEvents.cs +++ b/src/DurableTask.Core/Logging/LogEvents.cs @@ -1546,5 +1546,88 @@ public RenewActivityMessageFailed(TaskActivityWorkItem workItem, Exception excep Utils.AppName, Utils.PackageVersion); } + + internal class RenewOrchestrationWorkItemStarting : StructuredLogEvent, IEventSourceEvent + { + public RenewOrchestrationWorkItemStarting(TaskOrchestrationWorkItem workItem) + { + this.InstanceId = workItem.InstanceId; + } + + [StructuredLogField] + public string InstanceId { get; } + + public override EventId EventId => new EventId( + EventIds.RenewOrchestrationWorkItemStarting, + nameof(EventIds.RenewOrchestrationWorkItemStarting)); + + public override LogLevel Level => LogLevel.Debug; + + protected override string CreateLogMessage() => + $"{this.InstanceId}: Renewing orchestration work item"; + + void IEventSourceEvent.WriteEventSource() => + StructuredEventSource.Log.RenewOrchestrationWorkItemStarting( + this.InstanceId, + Utils.AppName, + Utils.PackageVersion); + } + + internal class RenewOrchestrationWorkItemCompleted : StructuredLogEvent, IEventSourceEvent + { + public RenewOrchestrationWorkItemCompleted(TaskOrchestrationWorkItem workItem) + { + this.InstanceId = workItem.InstanceId; + } + + [StructuredLogField] + public string InstanceId { get; } + + public override EventId EventId => new EventId( + EventIds.RenewOrchestrationWorkItemCompleted, + nameof(EventIds.RenewOrchestrationWorkItemCompleted)); + + public override LogLevel Level => LogLevel.Debug; + + protected override string CreateLogMessage() => + $"{this.InstanceId}: Renewed orchestration work item"; + + void IEventSourceEvent.WriteEventSource() => + StructuredEventSource.Log.RenewOrchestrationWorkItemCompleted( + this.InstanceId, + Utils.AppName, + Utils.PackageVersion); + } + + internal class RenewOrchestrationWorkItemFailed : StructuredLogEvent, IEventSourceEvent + { + public RenewOrchestrationWorkItemFailed(TaskOrchestrationWorkItem workItem, Exception exception) + { + this.InstanceId = workItem.InstanceId; + this.Details = exception.ToString(); + } + + [StructuredLogField] + public string InstanceId { get; } + + [StructuredLogField] + public string Details { get; } + + public override EventId EventId => new EventId( + EventIds.RenewOrchestrationWorkItemFailed, + nameof(EventIds.RenewOrchestrationWorkItemFailed)); + + public override LogLevel Level => LogLevel.Warning; + + protected override string CreateLogMessage() => + $"{this.InstanceId}: Failed to renew orchestration work item: {this.Details}"; + + void IEventSourceEvent.WriteEventSource() => + StructuredEventSource.Log.RenewOrchestrationWorkItemFailed( + this.InstanceId, + this.Details, + Utils.AppName, + Utils.PackageVersion); + } } } diff --git a/src/DurableTask.Core/Logging/LogHelper.cs b/src/DurableTask.Core/Logging/LogHelper.cs index 30615360f..f4d1ffe34 100644 --- a/src/DurableTask.Core/Logging/LogHelper.cs +++ b/src/DurableTask.Core/Logging/LogHelper.cs @@ -523,6 +523,44 @@ internal void DroppingOrchestrationMessage(TaskOrchestrationWorkItem workItem, T this.WriteStructuredLog(new LogEvents.DiscardingMessage(workItem, message, reason)); } } + + /// + /// Logs that an orchestration work item renewal operation is starting. + /// + /// The work item to be renewed. + internal void RenewOrchestrationWorkItemStarting(TaskOrchestrationWorkItem workItem) + { + if (this.IsStructuredLoggingEnabled) + { + this.WriteStructuredLog(new LogEvents.RenewOrchestrationWorkItemStarting(workItem)); + } + } + + /// + /// Logs that an orchestration work item renewal operation succeeded. + /// + /// The work item that was renewed. + internal void RenewOrchestrationWorkItemCompleted(TaskOrchestrationWorkItem workItem) + { + if (this.IsStructuredLoggingEnabled) + { + this.WriteStructuredLog(new LogEvents.RenewOrchestrationWorkItemCompleted(workItem)); + } + } + + /// + /// Logs that an orchestration work item renewal operation failed. + /// + /// The work item that was to be renewed. + /// The renew failure exception. + internal void RenewOrchestrationWorkItemFailed(TaskOrchestrationWorkItem workItem, Exception exception) + { + if (this.IsStructuredLoggingEnabled) + { + this.WriteStructuredLog(new LogEvents.RenewOrchestrationWorkItemFailed(workItem, exception), exception); + } + } + #endregion #region Activity dispatcher diff --git a/src/DurableTask.Core/Logging/StructuredEventSource.cs b/src/DurableTask.Core/Logging/StructuredEventSource.cs index f7c30354a..b9edc0a46 100644 --- a/src/DurableTask.Core/Logging/StructuredEventSource.cs +++ b/src/DurableTask.Core/Logging/StructuredEventSource.cs @@ -813,5 +813,57 @@ internal void FetchWorkItemFailure(string Dispatcher, string Details, string App ExtensionVersion); } } + + [Event(EventIds.RenewOrchestrationWorkItemStarting, Level = EventLevel.Verbose, Version = 1)] + internal void RenewOrchestrationWorkItemStarting( + string InstanceId, + string AppName, + string ExtensionVersion) + { + if (this.IsEnabled(EventLevel.Verbose)) + { + // TODO: Use WriteEventCore for better performance + this.WriteEvent( + EventIds.RenewOrchestrationWorkItemStarting, + InstanceId, + AppName, + ExtensionVersion); + } + } + + [Event(EventIds.RenewOrchestrationWorkItemCompleted, Level = EventLevel.Verbose, Version = 1)] + internal void RenewOrchestrationWorkItemCompleted( + string InstanceId, + string AppName, + string ExtensionVersion) + { + if (this.IsEnabled(EventLevel.Verbose)) + { + // TODO: Use WriteEventCore for better performance + this.WriteEvent( + EventIds.RenewOrchestrationWorkItemCompleted, + InstanceId, + AppName, + ExtensionVersion); + } + } + + [Event(EventIds.RenewOrchestrationWorkItemFailed, Level = EventLevel.Error, Version = 1)] + internal void RenewOrchestrationWorkItemFailed( + string InstanceId, + string Details, + string AppName, + string ExtensionVersion) + { + if (this.IsEnabled(EventLevel.Error)) + { + this.WriteEvent( + EventIds.RenewOrchestrationWorkItemFailed, + InstanceId, + Details, + AppName, + ExtensionVersion); + } + } } } diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 1bdc8dae7..d7e3dcc98 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -303,235 +303,262 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work OrchestrationState? instanceState = null; - - // Assumes that: if the batch contains a new "ExecutionStarted" event, it is the first message in the batch. - if (!this.ReconcileMessagesWithState(workItem)) + Task? renewTask = null; + using var renewCancellationTokenSource = new CancellationTokenSource(); + if (workItem.LockedUntilUtc < DateTime.MaxValue) { - // TODO : mark an orchestration as faulted if there is data corruption - this.logHelper.DroppingOrchestrationWorkItem(workItem, "Received work-item for an invalid orchestration"); - TraceHelper.TraceSession( - TraceEventType.Error, - "TaskOrchestrationDispatcher-DeletedOrchestration", - runtimeState.OrchestrationInstance?.InstanceId, - "Received work-item for an invalid orchestration"); - isCompleted = true; + // start a task to run RenewUntil + renewTask = Task.Factory.StartNew( + () => this.RenewUntil(workItem, renewCancellationTokenSource.Token), + renewCancellationTokenSource.Token); } - else + + try { - do + // Assumes that: if the batch contains a new "ExecutionStarted" event, it is the first message in the batch. + if (!this.ReconcileMessagesWithState(workItem)) { - continuedAsNew = false; - continuedAsNewMessage = null; - - this.logHelper.OrchestrationExecuting(runtimeState.OrchestrationInstance!, runtimeState.Name); - TraceHelper.TraceInstance( - TraceEventType.Verbose, - "TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin", - runtimeState.OrchestrationInstance, - "Executing user orchestration: {0}", - JsonDataConverter.Default.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true)); - - if (workItem.Cursor == null) - { - workItem.Cursor = await this.ExecuteOrchestrationAsync(runtimeState, workItem); - } - else + // TODO : mark an orchestration as faulted if there is data corruption + this.logHelper.DroppingOrchestrationWorkItem(workItem, "Received work-item for an invalid orchestration"); + TraceHelper.TraceSession( + TraceEventType.Error, + "TaskOrchestrationDispatcher-DeletedOrchestration", + runtimeState.OrchestrationInstance?.InstanceId, + "Received work-item for an invalid orchestration"); + isCompleted = true; + } + else + { + do { - await this.ResumeOrchestrationAsync(workItem); - } + continuedAsNew = false; + continuedAsNewMessage = null; - IReadOnlyList decisions = workItem.Cursor.LatestDecisions.ToList(); + this.logHelper.OrchestrationExecuting(runtimeState.OrchestrationInstance!, runtimeState.Name); + TraceHelper.TraceInstance( + TraceEventType.Verbose, + "TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin", + runtimeState.OrchestrationInstance, + "Executing user orchestration: {0}", + JsonDataConverter.Default.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true)); - this.logHelper.OrchestrationExecuted( - runtimeState.OrchestrationInstance!, - runtimeState.Name, - decisions); - TraceHelper.TraceInstance( - TraceEventType.Information, - "TaskOrchestrationDispatcher-ExecuteUserOrchestration-End", - runtimeState.OrchestrationInstance, - "Executed user orchestration. Received {0} orchestrator actions: {1}", - decisions.Count, - string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType))); + if (workItem.Cursor == null) + { + workItem.Cursor = await this.ExecuteOrchestrationAsync(runtimeState, workItem); + } + else + { + await this.ResumeOrchestrationAsync(workItem); + } - // TODO: Exception handling for invalid decisions, which is increasingly likely - // when custom middleware is involved (e.g. out-of-process scenarios). - foreach (OrchestratorAction decision in decisions) - { + IReadOnlyList decisions = workItem.Cursor.LatestDecisions.ToList(); + + this.logHelper.OrchestrationExecuted( + runtimeState.OrchestrationInstance!, + runtimeState.Name, + decisions); TraceHelper.TraceInstance( TraceEventType.Information, - "TaskOrchestrationDispatcher-ProcessOrchestratorAction", + "TaskOrchestrationDispatcher-ExecuteUserOrchestration-End", runtimeState.OrchestrationInstance, - "Processing orchestrator action of type {0}", - decision.OrchestratorActionType); - switch (decision.OrchestratorActionType) + "Executed user orchestration. Received {0} orchestrator actions: {1}", + decisions.Count, + string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType))); + + // TODO: Exception handling for invalid decisions, which is increasingly likely + // when custom middleware is involved (e.g. out-of-process scenarios). + foreach (OrchestratorAction decision in decisions) { - case OrchestratorActionType.ScheduleOrchestrator: - var scheduleTaskAction = (ScheduleTaskOrchestratorAction)decision; - var message = this.ProcessScheduleTaskDecision( - scheduleTaskAction, - runtimeState, - this.IncludeParameters); - messagesToSend.Add(message); - break; - case OrchestratorActionType.CreateTimer: - var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision; - timerMessages.Add(this.ProcessCreateTimerDecision( - timerOrchestratorAction, - runtimeState, - isInternal: false)); - break; - case OrchestratorActionType.CreateSubOrchestration: - var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; - orchestratorMessages.Add( - this.ProcessCreateSubOrchestrationInstanceDecision( - createSubOrchestrationAction, + TraceHelper.TraceInstance( + TraceEventType.Information, + "TaskOrchestrationDispatcher-ProcessOrchestratorAction", + runtimeState.OrchestrationInstance, + "Processing orchestrator action of type {0}", + decision.OrchestratorActionType); + switch (decision.OrchestratorActionType) + { + case OrchestratorActionType.ScheduleOrchestrator: + var scheduleTaskAction = (ScheduleTaskOrchestratorAction)decision; + var message = this.ProcessScheduleTaskDecision( + scheduleTaskAction, runtimeState, - this.IncludeParameters)); - break; - case OrchestratorActionType.SendEvent: - var sendEventAction = (SendEventOrchestratorAction)decision; - orchestratorMessages.Add( - this.ProcessSendEventDecision(sendEventAction, runtimeState)); - break; - case OrchestratorActionType.OrchestrationComplete: - OrchestrationCompleteOrchestratorAction completeDecision = (OrchestrationCompleteOrchestratorAction)decision; - TaskMessage? workflowInstanceCompletedMessage = - this.ProcessWorkflowCompletedTaskDecision(completeDecision, runtimeState, this.IncludeDetails, out continuedAsNew); - if (workflowInstanceCompletedMessage != null) - { - // Send complete message to parent workflow or to itself to start a new execution - // Store the event so we can rebuild the state - carryOverEvents = null; - if (continuedAsNew) + this.IncludeParameters); + messagesToSend.Add(message); + break; + case OrchestratorActionType.CreateTimer: + var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision; + timerMessages.Add(this.ProcessCreateTimerDecision( + timerOrchestratorAction, + runtimeState, + isInternal: false)); + break; + case OrchestratorActionType.CreateSubOrchestration: + var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; + orchestratorMessages.Add( + this.ProcessCreateSubOrchestrationInstanceDecision( + createSubOrchestrationAction, + runtimeState, + this.IncludeParameters)); + break; + case OrchestratorActionType.SendEvent: + var sendEventAction = (SendEventOrchestratorAction)decision; + orchestratorMessages.Add( + this.ProcessSendEventDecision(sendEventAction, runtimeState)); + break; + case OrchestratorActionType.OrchestrationComplete: + OrchestrationCompleteOrchestratorAction completeDecision = (OrchestrationCompleteOrchestratorAction)decision; + TaskMessage? workflowInstanceCompletedMessage = + this.ProcessWorkflowCompletedTaskDecision(completeDecision, runtimeState, this.IncludeDetails, out continuedAsNew); + if (workflowInstanceCompletedMessage != null) { - continuedAsNewMessage = workflowInstanceCompletedMessage; - continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent; - if (completeDecision.CarryoverEvents.Any()) + // Send complete message to parent workflow or to itself to start a new execution + // Store the event so we can rebuild the state + carryOverEvents = null; + if (continuedAsNew) { - carryOverEvents = completeDecision.CarryoverEvents.ToList(); - completeDecision.CarryoverEvents.Clear(); + continuedAsNewMessage = workflowInstanceCompletedMessage; + continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent; + if (completeDecision.CarryoverEvents.Any()) + { + carryOverEvents = completeDecision.CarryoverEvents.ToList(); + completeDecision.CarryoverEvents.Clear(); + } + } + else + { + orchestratorMessages.Add(workflowInstanceCompletedMessage); } } - else - { - orchestratorMessages.Add(workflowInstanceCompletedMessage); - } - } - - isCompleted = !continuedAsNew; - break; - default: - throw TraceHelper.TraceExceptionInstance( - TraceEventType.Error, - "TaskOrchestrationDispatcher-UnsupportedDecisionType", - runtimeState.OrchestrationInstance, - new NotSupportedException($"Decision type '{decision.OrchestratorActionType}' not supported")); - } - // Underlying orchestration service provider may have a limit of messages per call, to avoid the situation - // we keep on asking the provider if message count is ok and stop processing new decisions if not. - // - // We also put in a fake timer to force next orchestration task for remaining messages - int totalMessages = messagesToSend.Count + orchestratorMessages.Count + timerMessages.Count; - if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState)) - { - TraceHelper.TraceInstance( - TraceEventType.Information, - "TaskOrchestrationDispatcher-MaxMessageCountReached", - runtimeState.OrchestrationInstance, - "MaxMessageCount reached. Adding timer to process remaining events in next attempt."); + isCompleted = !continuedAsNew; + break; + default: + throw TraceHelper.TraceExceptionInstance( + TraceEventType.Error, + "TaskOrchestrationDispatcher-UnsupportedDecisionType", + runtimeState.OrchestrationInstance, + new NotSupportedException($"Decision type '{decision.OrchestratorActionType}' not supported")); + } - if (isCompleted || continuedAsNew) + // Underlying orchestration service provider may have a limit of messages per call, to avoid the situation + // we keep on asking the provider if message count is ok and stop processing new decisions if not. + // + // We also put in a fake timer to force next orchestration task for remaining messages + int totalMessages = messagesToSend.Count + orchestratorMessages.Count + timerMessages.Count; + if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState)) { TraceHelper.TraceInstance( TraceEventType.Information, - "TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted", + "TaskOrchestrationDispatcher-MaxMessageCountReached", runtimeState.OrchestrationInstance, - "Orchestration already completed. Skip adding timer for splitting messages."); + "MaxMessageCount reached. Adding timer to process remaining events in next attempt."); + + if (isCompleted || continuedAsNew) + { + TraceHelper.TraceInstance( + TraceEventType.Information, + "TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted", + runtimeState.OrchestrationInstance, + "Orchestration already completed. Skip adding timer for splitting messages."); + break; + } + + var dummyTimer = new CreateTimerOrchestratorAction + { + Id = FrameworkConstants.FakeTimerIdToSplitDecision, + FireAt = DateTime.UtcNow + }; + + timerMessages.Add(this.ProcessCreateTimerDecision( + dummyTimer, + runtimeState, + isInternal: true)); + isInterrupted = true; break; } - - var dummyTimer = new CreateTimerOrchestratorAction - { - Id = FrameworkConstants.FakeTimerIdToSplitDecision, - FireAt = DateTime.UtcNow - }; - - timerMessages.Add(this.ProcessCreateTimerDecision( - dummyTimer, - runtimeState, - isInternal: true)); - isInterrupted = true; - break; } - } - // correlation - CorrelationTraceClient.Propagate(() => { - if (runtimeState.ExecutionStartedEvent != null) - runtimeState.ExecutionStartedEvent.Correlation = CorrelationTraceContext.Current.SerializableTraceContext; - }); + // correlation + CorrelationTraceClient.Propagate(() => + { + if (runtimeState.ExecutionStartedEvent != null) + runtimeState.ExecutionStartedEvent.Correlation = CorrelationTraceContext.Current.SerializableTraceContext; + }); - // finish up processing of the work item - if (!continuedAsNew && runtimeState.Events.Last().EventType != EventType.OrchestratorCompleted) - { - runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); - } + // finish up processing of the work item + if (!continuedAsNew && runtimeState.Events.Last().EventType != EventType.OrchestratorCompleted) + { + runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); + } - if (isCompleted) - { - TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state"); - if (runtimeState.ExecutionStartedEvent != null) + if (isCompleted) { - instanceState = Utils.BuildOrchestrationState(runtimeState); + TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state"); + if (runtimeState.ExecutionStartedEvent != null) + { + instanceState = Utils.BuildOrchestrationState(runtimeState); + } } - } - else - { - if (continuedAsNew) + else { - TraceHelper.TraceSession( - TraceEventType.Information, - "TaskOrchestrationDispatcher-UpdatingStateForContinuation", - workItem.InstanceId, - "Updating state for continuation"); - - // correlation - CorrelationTraceClient.Propagate(() => + if (continuedAsNew) { - continueAsNewExecutionStarted!.Correlation = CorrelationTraceContext.Current.SerializableTraceContext; - }); + TraceHelper.TraceSession( + TraceEventType.Information, + "TaskOrchestrationDispatcher-UpdatingStateForContinuation", + workItem.InstanceId, + "Updating state for continuation"); - runtimeState = new OrchestrationRuntimeState(); - runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); - runtimeState.AddEvent(continueAsNewExecutionStarted!); - runtimeState.Status = workItem.OrchestrationRuntimeState.Status ?? carryOverStatus; - carryOverStatus = workItem.OrchestrationRuntimeState.Status; + // correlation + CorrelationTraceClient.Propagate(() => + { + continueAsNewExecutionStarted!.Correlation = CorrelationTraceContext.Current.SerializableTraceContext; + }); - if (carryOverEvents != null) - { - foreach (var historyEvent in carryOverEvents) + runtimeState = new OrchestrationRuntimeState(); + runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); + runtimeState.AddEvent(continueAsNewExecutionStarted!); + runtimeState.Status = workItem.OrchestrationRuntimeState.Status ?? carryOverStatus; + carryOverStatus = workItem.OrchestrationRuntimeState.Status; + + if (carryOverEvents != null) { - runtimeState.AddEvent(historyEvent); + foreach (var historyEvent in carryOverEvents) + { + runtimeState.AddEvent(historyEvent); + } } - } - runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); - workItem.OrchestrationRuntimeState = runtimeState; + runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); + workItem.OrchestrationRuntimeState = runtimeState; - if (workItem.LockedUntilUtc < DateTime.UtcNow.AddMinutes(1)) - { - await this.orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem); + workItem.Cursor = null; } - workItem.Cursor = null; + instanceState = Utils.BuildOrchestrationState(runtimeState); } - - instanceState = Utils.BuildOrchestrationState(runtimeState); + } while (continuedAsNew); + } + } + finally + { + if (renewTask != null) + { + try + { + renewCancellationTokenSource.Cancel(); + await renewTask; + } + catch (ObjectDisposedException) + { + // ignore + } + catch (OperationCanceledException) + { + // ignore } - } while (continuedAsNew); + } } if (workItem.RestoreOriginalRuntimeStateDuringCompletion) @@ -570,6 +597,46 @@ static OrchestrationExecutionContext GetOrchestrationExecutionContext(Orchestrat return new OrchestrationExecutionContext { OrchestrationTags = runtimeState.Tags ?? new Dictionary(capacity: 0) }; } + TimeSpan MinRenewalInterval = TimeSpan.FromSeconds(5); // prevents excessive retries if clocks are off + TimeSpan MaxRenewalInterval = TimeSpan.FromSeconds(30); + + async Task RenewUntil(TaskOrchestrationWorkItem workItem, CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + TimeSpan delay = workItem.LockedUntilUtc - DateTime.UtcNow - TimeSpan.FromSeconds(30); + if (delay < MinRenewalInterval) + { + delay = MinRenewalInterval; + } + else if (delay > MaxRenewalInterval) + { + delay = MaxRenewalInterval; + } + + await Utils.DelayWithCancellation(delay, cancellationToken); + + if (cancellationToken.IsCancellationRequested) + { + return; + } + + try + { + this.logHelper.RenewOrchestrationWorkItemStarting(workItem); + TraceHelper.Trace(TraceEventType.Information, "TaskOrchestrationDispatcher-RenewWorkItemStarting", "Renewing work item for instance {0}", workItem.InstanceId); + await this.orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem); + this.logHelper.RenewOrchestrationWorkItemCompleted(workItem); + TraceHelper.Trace(TraceEventType.Information, "TaskOrchestrationDispatcher-RenewWorkItemCompleted", "Successfully renewed work item for instance {0}", workItem.InstanceId); + } + catch (Exception exception) when (!Utils.IsFatal(exception)) + { + this.logHelper.RenewOrchestrationWorkItemFailed(workItem, exception); + TraceHelper.TraceException(TraceEventType.Warning, "TaskOrchestrationDispatcher-RenewWorkItemFailed", exception, "Failed to renew work item for instance {0}", workItem.InstanceId); + } + } + } + async Task ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState, TaskOrchestrationWorkItem workItem) { // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never diff --git a/src/DurableTask.Emulator/DurableTask.Emulator.csproj b/src/DurableTask.Emulator/DurableTask.Emulator.csproj index 9e5383468..e165bbe31 100644 --- a/src/DurableTask.Emulator/DurableTask.Emulator.csproj +++ b/src/DurableTask.Emulator/DurableTask.Emulator.csproj @@ -13,7 +13,7 @@ - + diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageOrchestrationServiceSettingsTest.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageOrchestrationServiceSettingsTest.cs index a63a7b332..ede96b597 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageOrchestrationServiceSettingsTest.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageOrchestrationServiceSettingsTest.cs @@ -21,10 +21,12 @@ public class AzureStorageOrchestrationServiceSettingsTest [TestMethod] public void TrackingStoreStorageAccountDetailsNotSet() { - var settings = new AzureStorageOrchestrationServiceSettings() + var settings = new AzureStorageOrchestrationServiceSettings { TaskHubName = "foo", }; + + Assert.IsFalse(settings.HasTrackingStoreStorageAccount); Assert.AreEqual("fooHistory", settings.HistoryTableName); Assert.AreEqual("fooInstances", settings.InstanceTableName); } @@ -32,12 +34,14 @@ public void TrackingStoreStorageAccountDetailsNotSet() [TestMethod] public void TrackingStoreStorageAccountDetailsHasSet() { - var settings = new AzureStorageOrchestrationServiceSettings() + var settings = new AzureStorageOrchestrationServiceSettings { TaskHubName = "foo", TrackingStoreNamePrefix = "bar", - TrackingServiceClientProvider = StorageServiceClientProvider.ForTable("connectionString"), + TrackingServiceClientProvider = new TrackingServiceClientProvider("connectionString"), }; + + Assert.IsTrue(settings.HasTrackingStoreStorageAccount); Assert.AreEqual("barHistory", settings.HistoryTableName); Assert.AreEqual("barInstances", settings.InstanceTableName); } diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 8cd1a3682..9749d8d46 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -147,6 +147,35 @@ public async Task ParentOfSequentialOrchestration() } } + /// + /// End-to-end test which runs a slow orchestrator that causes work item renewal + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task LongRunningOrchestrator(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost( + enableExtendedSessions, + modifySettingsAction: (AzureStorageOrchestrationServiceSettings settings) => + { + // set a short timeout so we can test that the renewal works + settings.ControlQueueVisibilityTimeout = TimeSpan.FromSeconds(10); + })) + { + await host.StartAsync(); + + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.LongRunningOrchestrator), "0"); + var status = await client.WaitForCompletionAsync(StandardTimeout); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + Assert.AreEqual("ok", JToken.Parse(status?.Output)); + + await host.StopAsync(); + } + } + + [TestMethod] public async Task GetAllOrchestrationStatuses() { @@ -1664,6 +1693,33 @@ public async Task LargeTextMessagePayloads_FetchLargeMessages_QueryState(bool en } } + /// + /// End-to-end test which validates that exception messages that are considered valid Urls in the Uri.TryCreate() method + /// are handled with an additional Uri format check + /// + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task LargeTextMessagePayloads_URIFormatCheck(bool enableExtendedSessions) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions, fetchLargeMessages: true)) + { + await host.StartAsync(); + + string message = this.GenerateMediumRandomStringPayload().ToString(); + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.ThrowException), "durabletask.core.exceptions.taskfailedexception: Task failed with an unhandled exception: This is an invalid operation.)"); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + + //Ensure that orchestration state querying also retrieves messages + status = (await client.GetStateAsync(status.OrchestrationInstance.InstanceId)).First(); + + Assert.AreEqual(OrchestrationStatus.Failed, status?.OrchestrationStatus); + Assert.IsTrue(status?.Output.Contains("invalid operation") == true); + + await host.StopAsync(); + } + } + private StringBuilder GenerateMediumRandomStringPayload(int numChars = 128*1024, short utf8ByteSize = 1, short utf16ByteSize = 2) { string Chars; @@ -2274,6 +2330,23 @@ public override Task RunTask(OrchestrationContext context, int n) } } + internal class LongRunningOrchestrator : TaskOrchestration + { + public override Task RunTask(OrchestrationContext context, string input) + { + Thread.Sleep(TimeSpan.FromSeconds(10)); + if (input == "0") + { + context.ContinueAsNew("1"); + return Task.FromResult(""); + } + else + { + return Task.FromResult("ok"); + } + } + } + [KnownType(typeof(Activities.GetFileList))] [KnownType(typeof(Activities.GetFileSize))] internal class DiskUsage : TaskOrchestration @@ -2640,6 +2713,24 @@ public override void OnEvent(OrchestrationContext context, string name, bool app } } + [KnownType(typeof(Activities.Throw))] + internal class ThrowException : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext context, string message) + { + if (string.IsNullOrEmpty(message)) + { + // This throw happens directly in the orchestration. + throw new Exception(message); + } + + // This throw happens in the implementation of an activity. + await context.ScheduleTask(typeof(Activities.Throw), message); + return null; + + } + } + [KnownType(typeof(Activities.Throw))] internal class Throw : TaskOrchestration { diff --git a/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj b/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj index c01d2c199..73fff3c3f 100644 --- a/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj +++ b/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj @@ -16,7 +16,7 @@ - + diff --git a/test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj b/test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj index 061e20f2a..3800b3898 100644 --- a/test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj +++ b/test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj @@ -18,7 +18,7 @@ - + diff --git a/test/DurableTask.Test.Orchestrations/DurableTask.Test.Orchestrations.csproj b/test/DurableTask.Test.Orchestrations/DurableTask.Test.Orchestrations.csproj index 0e1784357..e2e94505d 100644 --- a/test/DurableTask.Test.Orchestrations/DurableTask.Test.Orchestrations.csproj +++ b/test/DurableTask.Test.Orchestrations/DurableTask.Test.Orchestrations.csproj @@ -6,7 +6,7 @@ - + From 8c71de33d416de17636eb0c1fb1f879b26a06a5a Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Fri, 14 Apr 2023 11:02:13 -0700 Subject: [PATCH 04/62] Fix Uri for sastokens --- .../Storage/TableClientExtensionsTests.cs | 7 ++++++- .../Storage/TableClientExtensions.cs | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs b/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs index 430d91ce3..307a8137e 100644 --- a/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs +++ b/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs @@ -45,7 +45,12 @@ public void GetUri_ServiceEndpoint() [TestMethod] public void GetUri_TableEndpoint() { - var client = new TableClient(new Uri("https://foo.table.core.windows.net/bar")); + TableClient client; + + client = new TableClient(new Uri("https://foo.table.core.windows.net/bar")); + Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri()); + + client = new TableClient(new Uri("https://foo.table.core.windows.net/bar?sv=2019-12-12&ss=t&srt=s&sp=rwdlacu&se=2020-08-28T23:45:30Z&st=2020-08-26T15:45:30Z&spr=https&sig=mySig&tn=someTableName")); Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri()); } } diff --git a/src/DurableTask.AzureStorage/Storage/TableClientExtensions.cs b/src/DurableTask.AzureStorage/Storage/TableClientExtensions.cs index d3a77d17b..da2b73b1f 100644 --- a/src/DurableTask.AzureStorage/Storage/TableClientExtensions.cs +++ b/src/DurableTask.AzureStorage/Storage/TableClientExtensions.cs @@ -28,7 +28,9 @@ static class TableClientExtensions } Uri? endpoint = GetEndpointFunc(tableClient); - return endpoint != null ? new TableUriBuilder(endpoint) { Tablename = tableClient.Name }.ToUri() : null; + return endpoint != null + ? new TableUriBuilder(endpoint) { Query = null, Sas = null, Tablename = tableClient.Name }.ToUri() + : null; } static readonly Func GetEndpointFunc = CreateGetEndpointFunc(); From 34e4a2a35c4122f88b27536fb078e2de60a2161c Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Fri, 14 Apr 2023 11:10:00 -0700 Subject: [PATCH 05/62] nit: table name --- .../Storage/TableClientExtensionsTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs b/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs index 307a8137e..31bdd4d85 100644 --- a/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs +++ b/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs @@ -50,7 +50,7 @@ public void GetUri_TableEndpoint() client = new TableClient(new Uri("https://foo.table.core.windows.net/bar")); Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri()); - client = new TableClient(new Uri("https://foo.table.core.windows.net/bar?sv=2019-12-12&ss=t&srt=s&sp=rwdlacu&se=2020-08-28T23:45:30Z&st=2020-08-26T15:45:30Z&spr=https&sig=mySig&tn=someTableName")); + client = new TableClient(new Uri("https://foo.table.core.windows.net/bar?sv=2019-12-12&ss=t&srt=s&sp=rwdlacu&se=2020-08-28T23:45:30Z&st=2020-08-26T15:45:30Z&spr=https&sig=mySig&tn=bar")); Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri()); } } From 3988e06a0514124fbe71ee9baef67cdde37f227a Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Sat, 22 Apr 2023 13:33:05 -0700 Subject: [PATCH 06/62] Debugging --- src/DurableTask.AzureStorage/AssemblyInfo.cs | 2 -- .../DurableTask.AzureStorage.csproj | 2 ++ src/DurableTask.Core/AssemblyInfo.cs | 6 ------ src/DurableTask.Core/DurableTask.Core.csproj | 2 ++ 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/DurableTask.AzureStorage/AssemblyInfo.cs b/src/DurableTask.AzureStorage/AssemblyInfo.cs index 15255eaa0..bbb658705 100644 --- a/src/DurableTask.AzureStorage/AssemblyInfo.cs +++ b/src/DurableTask.AzureStorage/AssemblyInfo.cs @@ -14,6 +14,4 @@ using System.Runtime.CompilerServices; #if !SIGN_ASSEMBLY -[assembly: InternalsVisibleTo("DurableTask.AzureStorage.Tests")] -[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] #endif \ No newline at end of file diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 7acdc2ba7..b7c77ede1 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -32,6 +32,8 @@ $(MajorVersion).$(MinorVersion).0.0 $(VersionPrefix)-$(VersionSuffix) + True + F:\Git\_forks\durabletask\tools\sign.snk diff --git a/src/DurableTask.Core/AssemblyInfo.cs b/src/DurableTask.Core/AssemblyInfo.cs index d414ce1e6..b65c15f70 100644 --- a/src/DurableTask.Core/AssemblyInfo.cs +++ b/src/DurableTask.Core/AssemblyInfo.cs @@ -12,9 +12,3 @@ // ---------------------------------------------------------------------------------- using System.Runtime.CompilerServices; - -#if !SIGN_ASSEMBLY -[assembly: InternalsVisibleTo("DurableTask.Core.Tests")] -[assembly: InternalsVisibleTo("DurableTask.Framework.Tests")] -[assembly: InternalsVisibleTo("DurableTask.ServiceBus.Tests")] -#endif diff --git a/src/DurableTask.Core/DurableTask.Core.csproj b/src/DurableTask.Core/DurableTask.Core.csproj index 8c1983c72..758938f6d 100644 --- a/src/DurableTask.Core/DurableTask.Core.csproj +++ b/src/DurableTask.Core/DurableTask.Core.csproj @@ -28,6 +28,8 @@ $(MajorVersion).$(MinorVersion).0.0 $(VersionPrefix) + True + F:\Git\_forks\durabletask\tools\sign.snk From cd477c749c71fd4fd16aaf3a245c0c145c5ac975 Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Wed, 31 May 2023 13:41:37 -0700 Subject: [PATCH 07/62] Merge Main into Azure Storage v12 Branch (#911) --- .../DurableTask.AzureServiceFabric.csproj | 2 +- .../FabricOrchestrationService.cs | 127 ++++++++++++++---- .../FabricOrchestrationServiceClient.cs | 51 +++++-- .../RemoteOrchestrationServiceClient.cs | 4 +- .../Stores/SessionProvider.cs | 41 ++++-- src/DurableTask.AzureServiceFabric/Utils.cs | 61 ++++++--- .../AzureStorageOrchestrationService.cs | 3 +- .../MessageManager.cs | 6 + .../Messaging/OrchestrationSession.cs | 4 + .../OrchestrationSessionManager.cs | 3 + .../Tracking/AzureTableTrackingStore.cs | 54 ++++++-- .../Tracking/ITrackingStore.cs | 3 +- .../InstanceStoreBackedTrackingStore.cs | 2 +- .../Tracking/OrchestrationHistory.cs | 8 ++ .../Tracking/TrackingStoreBase.cs | 2 +- .../FunctionalTests.cs | 111 ++++++++++++++- 16 files changed, 401 insertions(+), 81 deletions(-) diff --git a/src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj b/src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj index 619395503..5666c8b87 100644 --- a/src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj +++ b/src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj @@ -6,7 +6,7 @@ true Microsoft.Azure.DurableTask.AzureServiceFabric true - 2.3.7 + 2.3.9 $(Version) $(Version) Azure Service Fabric provider extension for the Durable Task Framework. diff --git a/src/DurableTask.AzureServiceFabric/FabricOrchestrationService.cs b/src/DurableTask.AzureServiceFabric/FabricOrchestrationService.cs index dd71ccec4..8f6c46d1c 100644 --- a/src/DurableTask.AzureServiceFabric/FabricOrchestrationService.cs +++ b/src/DurableTask.AzureServiceFabric/FabricOrchestrationService.cs @@ -28,6 +28,7 @@ namespace DurableTask.AzureServiceFabric using DurableTask.AzureServiceFabric.TaskHelpers; using DurableTask.AzureServiceFabric.Tracing; using Microsoft.ServiceFabric.Data; + using Newtonsoft.Json; class FabricOrchestrationService : IOrchestrationService { @@ -149,6 +150,7 @@ public async Task LockNextTaskOrchestrationWorkItemAs newMessages = await this.orchestrationProvider.ReceiveSessionMessagesAsync(currentSession); var currentRuntimeState = new OrchestrationRuntimeState(currentSession.SessionState); + var workItem = new TaskOrchestrationWorkItem() { NewMessages = newMessages.Select(m => m.Value.TaskMessage).ToList(), @@ -167,7 +169,7 @@ public async Task LockNextTaskOrchestrationWorkItemAs bool isComplete = this.IsOrchestrationComplete(currentRuntimeState.OrchestrationStatus); if (isComplete) { - await this.HandleCompletedOrchestration(workItem); + await this.HandleCompletedOrchestrationAsync(workItem); } this.orchestrationProvider.TryUnlockSession(currentSession.SessionId, isComplete: isComplete); @@ -209,10 +211,27 @@ public Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem wo OrchestrationState orchestrationState) { SessionInformation sessionInfo = GetSessionInfo(workItem.InstanceId); - ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId, + bool isComplete = false; + + try + { + var orchestrationStatus = workItem.OrchestrationRuntimeState.OrchestrationStatus; + ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId, workItem.OrchestrationRuntimeState.OrchestrationInstance?.ExecutionId, - $"Current orchestration status: {workItem.OrchestrationRuntimeState.OrchestrationStatus}"); - bool isComplete = this.IsOrchestrationComplete(workItem.OrchestrationRuntimeState.OrchestrationStatus); + $"Current orchestration status: {orchestrationStatus}"); + isComplete = this.IsOrchestrationComplete(orchestrationStatus); + } + catch (InvalidOperationException ex) + { + // OrchestrationRuntimeState.OrchestrationStatus throws 'InvalidOperationException' if 'ExecutionStartedEvent' is missing. + // Do not process the orchestration workitem if 'ExecutionStartedEvent' is missing. + // This can happen when an orchestration message like ExecutionTerminatedEvent is sent to an already finished orchestration + if (workItem.OrchestrationRuntimeState.ExecutionStartedEvent == null) + { + ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition($"InstanceId: '{workItem.InstanceId}', exception: {ex}. Dropping the bad orchestration to avoid noise."); + await this.DropOrchestrationAsync(workItem); + } + } IList sessionsToEnqueue = null; List> scheduledMessages = null; @@ -271,7 +290,7 @@ public Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem wo if (workItem.OrchestrationRuntimeState.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew) { - await HandleCompletedOrchestration(workItem); + await HandleCompletedOrchestrationAsync(workItem); } // When an orchestration is completed, we need to drop the session which involves 2 steps (1) Removing the row from sessions @@ -342,12 +361,37 @@ public Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem wo if (isComplete) { - await HandleCompletedOrchestration(workItem); + await HandleCompletedOrchestrationAsync(workItem); } } + async Task DropOrchestrationAsync(TaskOrchestrationWorkItem workItem) + { + await CompleteOrchestrationAsync(workItem); + + string message = $"{nameof(DropOrchestrationAsync)}: Dropped. Orchestration history: {JsonConvert.SerializeObject(workItem.OrchestrationRuntimeState.Events)}"; + ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId, + workItem.OrchestrationRuntimeState.OrchestrationInstance?.ExecutionId, + message); + } + // Caller should ensure the workItem has reached terminal state. - private async Task HandleCompletedOrchestration(TaskOrchestrationWorkItem workItem) + async Task HandleCompletedOrchestrationAsync(TaskOrchestrationWorkItem workItem) + { + await CompleteOrchestrationAsync(workItem); + + string message = string.Format("Orchestration with instanceId : '{0}' and executionId : '{1}' Finished with the status {2} and result {3} in {4} seconds.", + workItem.InstanceId, + workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId, + workItem.OrchestrationRuntimeState.OrchestrationStatus.ToString(), + workItem.OrchestrationRuntimeState.Output, + (workItem.OrchestrationRuntimeState.CompletedTime - workItem.OrchestrationRuntimeState.CreatedTime).TotalSeconds); + ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId, + workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId, + message); + } + + async Task CompleteOrchestrationAsync(TaskOrchestrationWorkItem workItem) { await RetryHelper.ExecuteWithRetryOnTransient(async () => { @@ -357,35 +401,34 @@ private async Task HandleCompletedOrchestration(TaskOrchestrationWorkItem workIt { new OrchestrationStateInstanceEntity() { - State = Utils.BuildOrchestrationState(workItem.OrchestrationRuntimeState) + State = Utils.BuildOrchestrationState(workItem) } }); + + var instance = workItem.OrchestrationRuntimeState.OrchestrationInstance; + if (instance == null) + { + // This condition happens when an orchestration message like ExecutionTerminatedEvent enqueued for an already completed orchestration + SessionInformation sessionInfo = this.GetSessionInfo(workItem.InstanceId); + instance = sessionInfo.Instance; + } + // DropSession does 2 things (like mentioned in the comments above) - remove the row from sessions dictionary // and delete the session messages dictionary. The second step is in a background thread and not part of transaction. // However even if this transaction failed but we ended up deleting session messages dictionary, that's ok - at // that time, it should be an empty dictionary and we would have updated the runtime session state to full completed // state in the transaction from Complete method. So the subsequent attempt would be able to complete the session. - await this.orchestrationProvider.DropSession(txn, workItem.OrchestrationRuntimeState.OrchestrationInstance); + await this.orchestrationProvider.DropSessionAsync(txn, instance); await txn.CommitAsync(); } - }, uniqueActionIdentifier: $"OrchestrationId = '{workItem.InstanceId}', Action = '{nameof(HandleCompletedOrchestration)}'"); + }, uniqueActionIdentifier: $"OrchestrationId = '{workItem.InstanceId}', Action = '{nameof(CompleteOrchestrationAsync)}'"); this.instanceStore.OnOrchestrationCompleted(workItem.OrchestrationRuntimeState.OrchestrationInstance); - - string message = string.Format("Orchestration with instanceId : '{0}' and executionId : '{1}' Finished with the status {2} and result {3} in {4} seconds.", - workItem.InstanceId, - workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId, - workItem.OrchestrationRuntimeState.OrchestrationStatus.ToString(), - workItem.OrchestrationRuntimeState.Output, - (workItem.OrchestrationRuntimeState.CompletedTime - workItem.OrchestrationRuntimeState.CreatedTime).TotalSeconds); - ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId, - workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId, - message); } public Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { - SessionInformation sessionInfo = TryRemoveSessionInfo(workItem.InstanceId); + SessionInformation sessionInfo = this.TryRemoveSessionInfo(workItem.InstanceId); if (sessionInfo == null) { ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition($"{nameof(AbandonTaskOrchestrationWorkItemAsync)} : Could not get a session info object while trying to abandon session {workItem.InstanceId}"); @@ -399,9 +442,23 @@ public Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem work public Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { - bool isComplete = this.IsOrchestrationComplete(workItem.OrchestrationRuntimeState.OrchestrationStatus); + bool isComplete = false; + try + { + isComplete = this.IsOrchestrationComplete(workItem.OrchestrationRuntimeState.OrchestrationStatus); + } + catch (InvalidOperationException ex) + { + // OrchestrationRuntimeState.OrchestrationStatus throws 'InvalidOperationException' if 'ExecutionStartedEvent' is missing. + // This can happen when an orchestration message like ExecutionTerminatedEvent is sent to an already finished orchestration + if (workItem.OrchestrationRuntimeState.ExecutionStartedEvent == null) + { + ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition($"InstanceId: '{workItem.InstanceId}', exception: {ex}. Dropping/Unlocking the session as completed."); + isComplete = true; + } + } - SessionInformation sessionInfo = TryRemoveSessionInfo(workItem.InstanceId); + SessionInformation sessionInfo = this.TryRemoveSessionInfo(workItem.InstanceId); if (sessionInfo != null) { this.orchestrationProvider.TryUnlockSession(sessionInfo.Instance, isComplete: isComplete); @@ -472,10 +529,28 @@ public async Task CompleteTaskActivityWorkItemAsync(TaskActivityWorkItem workIte } } - public Task AbandonTaskActivityWorkItemAsync(TaskActivityWorkItem workItem) + public async Task AbandonTaskActivityWorkItemAsync(TaskActivityWorkItem workItem) { - this.activitiesProvider.Abandon(workItem.Id); - return Task.CompletedTask; + bool removed = false; + using (var txn = this.stateManager.CreateTransaction()) + { + // Remove task activity if orchestration was already terminated or cleaned up + if (!await this.orchestrationProvider.ContainsSessionAsync(txn, workItem.TaskMessage.OrchestrationInstance)) + { + var errorMessage = $"Session doesn't exist. Dropping TaskActivity for Orchestration = '{workItem.TaskMessage.OrchestrationInstance}', ActivityId = '{workItem.Id}', Action = '{nameof(AbandonTaskActivityWorkItemAsync)}'"; + ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition(errorMessage); + await this.activitiesProvider.CompleteAsync(txn, workItem.Id); + removed = true; + } + + await txn.CommitAsync(); + } + + if (!removed) + { + // Re-Enqueue task activity + this.activitiesProvider.Abandon(workItem.Id); + } } public Task RenewTaskActivityWorkItemLockAsync(TaskActivityWorkItem workItem) diff --git a/src/DurableTask.AzureServiceFabric/FabricOrchestrationServiceClient.cs b/src/DurableTask.AzureServiceFabric/FabricOrchestrationServiceClient.cs index 6f4788824..73f0171e7 100644 --- a/src/DurableTask.AzureServiceFabric/FabricOrchestrationServiceClient.cs +++ b/src/DurableTask.AzureServiceFabric/FabricOrchestrationServiceClient.cs @@ -18,19 +18,16 @@ namespace DurableTask.AzureServiceFabric using System.Linq; using System.Threading; using System.Threading.Tasks; - + using DurableTask.AzureServiceFabric.Stores; + using DurableTask.AzureServiceFabric.TaskHelpers; + using DurableTask.AzureServiceFabric.Tracing; using DurableTask.Core; using DurableTask.Core.Exceptions; using DurableTask.Core.History; + using DurableTask.Core.Serializing; using DurableTask.Core.Tracking; - using DurableTask.AzureServiceFabric.Stores; - using DurableTask.AzureServiceFabric.TaskHelpers; - using DurableTask.AzureServiceFabric.Tracing; - using Microsoft.ServiceFabric.Data; - using Newtonsoft.Json; - using DurableTask.Core.Serializing; class FabricOrchestrationServiceClient : IOrchestrationServiceClient, IFabricProviderClient { @@ -124,28 +121,56 @@ public async Task ForceTerminateTaskOrchestrationAsync(string instanceId, string if (latestExecutionId == null) { - throw new ArgumentException($"No execution id found for given instanceId {instanceId}, can only terminate the latest execution of a given orchestration"); + throw new InvalidOperationException($"No execution id found for given instanceId {instanceId}, can only terminate the latest execution of a given orchestration"); } + var orchestrationInstance = new OrchestrationInstance { InstanceId = instanceId, ExecutionId = latestExecutionId }; if (reason?.Trim().StartsWith("CleanupStore", StringComparison.OrdinalIgnoreCase) == true) { using (var txn = this.stateManager.CreateTransaction()) { - // DropSession does 2 things (like mentioned in the comments above) - remove the row from sessions dictionary - // and delete the session messages dictionary. The second step is in a background thread and not part of transaction. + var stateInstance = await this.instanceStore.GetOrchestrationStateAsync(instanceId, latestExecutionId); + var state = stateInstance?.State; + if (state == null) + { + state = new OrchestrationState() + { + OrchestrationInstance = orchestrationInstance, + LastUpdatedTime = DateTime.UtcNow, + }; + } + + state.OrchestrationStatus = OrchestrationStatus.Terminated; + state.Output = $"Orchestration dropped with reason '{reason}'"; + + await this.instanceStore.WriteEntitiesAsync(txn, new InstanceEntityBase[] + { + new OrchestrationStateInstanceEntity() + { + State = state + } + }); ; + // DropSession does 2 things : removes the row from sessions dictionary and delete the session messages dictionary. + // The second step is in a background thread and not part of transaction. // However even if this transaction failed but we ended up deleting session messages dictionary, that's ok - at // that time, it should be an empty dictionary and we would have updated the runtime session state to full completed // state in the transaction from Complete method. So the subsequent attempt would be able to complete the session. - var instance = new OrchestrationInstance { InstanceId = instanceId, ExecutionId = latestExecutionId }; - await this.orchestrationProvider.DropSession(txn, instance); + await this.orchestrationProvider.DropSessionAsync(txn, orchestrationInstance); await txn.CommitAsync(); + + // TODO: Renmove from FabricOrchestrationService.SessionInfo dictionary and SessionProvider.lockedSessions } + + this.instanceStore.OnOrchestrationCompleted(orchestrationInstance); + + string message = $"{nameof(ForceTerminateTaskOrchestrationAsync)}: Terminated with reason '{reason}'"; + ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(instanceId, latestExecutionId, message); } else { var taskMessage = new TaskMessage { - OrchestrationInstance = new OrchestrationInstance { InstanceId = instanceId, ExecutionId = latestExecutionId }, + OrchestrationInstance = orchestrationInstance, Event = new ExecutionTerminatedEvent(-1, reason) }; diff --git a/src/DurableTask.AzureServiceFabric/Remote/RemoteOrchestrationServiceClient.cs b/src/DurableTask.AzureServiceFabric/Remote/RemoteOrchestrationServiceClient.cs index 8424a83d0..04f85e8e4 100644 --- a/src/DurableTask.AzureServiceFabric/Remote/RemoteOrchestrationServiceClient.cs +++ b/src/DurableTask.AzureServiceFabric/Remote/RemoteOrchestrationServiceClient.cs @@ -31,6 +31,7 @@ namespace DurableTask.AzureServiceFabric.Remote using Newtonsoft.Json; using Newtonsoft.Json.Linq; + using System.Web.Http.Results; /// /// Allows to interact with a remote IOrchestrationServiceClient @@ -122,7 +123,8 @@ public async Task ForceTerminateTaskOrchestrationAsync(string instanceId, string { if (!response.IsSuccessStatusCode) { - throw new RemoteServiceException("Unable to terminate task instance", response.StatusCode); + var message = await response.Content.ReadAsStringAsync(); + throw new RemoteServiceException($"Unable to terminate task instance. Error: {response.StatusCode}:{message}", response.StatusCode); } } } diff --git a/src/DurableTask.AzureServiceFabric/Stores/SessionProvider.cs b/src/DurableTask.AzureServiceFabric/Stores/SessionProvider.cs index 8637d5559..bb3dea4ee 100644 --- a/src/DurableTask.AzureServiceFabric/Stores/SessionProvider.cs +++ b/src/DurableTask.AzureServiceFabric/Stores/SessionProvider.cs @@ -20,11 +20,9 @@ namespace DurableTask.AzureServiceFabric.Stores using System.Linq; using System.Threading; using System.Threading.Tasks; - - using DurableTask.Core; using DurableTask.AzureServiceFabric.TaskHelpers; using DurableTask.AzureServiceFabric.Tracing; - + using DurableTask.Core; using Microsoft.ServiceFabric.Data; /// @@ -87,6 +85,7 @@ public async Task AcceptSessionAsync(TimeSpan receiveTimeout) if (!IsStopped()) { bool newItemsBeforeTimeout = true; + bool reEnqueueOnException = true; while (newItemsBeforeTimeout) { if (this.fetchQueue.TryDequeue(out string returnInstanceId)) @@ -108,15 +107,17 @@ public async Task AcceptSessionAsync(TimeSpan receiveTimeout) } else { - var errorMessage = $"Internal Server Error : Unexpected to dequeue the session {returnInstanceId} which was already locked before"; + var errorMessage = $"Internal Server Error : Unexpected to dequeue the session {returnInstanceId} which was already locked before. Do not re-enture the item-without-session"; ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition(errorMessage); throw new Exception(errorMessage); } } else { + // If the session state is cleared using ForceTerminateOrchestration + reason (cleanupstore) then we may end up with a stale entry in the queue var errorMessage = $"Internal Server Error: Did not find the session object in reliable dictionary while having the session {returnInstanceId} in memory"; ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition(errorMessage); + reEnqueueOnException = false; throw new Exception(errorMessage); } } @@ -124,7 +125,10 @@ public async Task AcceptSessionAsync(TimeSpan receiveTimeout) } catch (Exception) { - this.fetchQueue.Enqueue(returnInstanceId); + if (reEnqueueOnException) + { + this.fetchQueue.Enqueue(returnInstanceId); + } throw; } } @@ -165,15 +169,23 @@ public async Task UpdateSessionState(ITransaction transaction, OrchestrationInst public async Task AppendMessageAsync(TaskMessageItem newMessage) { - await RetryHelper.ExecuteWithRetryOnTransient(async () => + bool added = await RetryHelper.ExecuteWithRetryOnTransient(async () => { + bool added = false; using (var txn = this.StateManager.CreateTransaction()) { - await this.AppendMessageAsync(txn, newMessage); + added = await this.TryAppendMessageAsync(txn, newMessage); await txn.CommitAsync(); } + + return added; }, uniqueActionIdentifier: $"Orchestration = '{newMessage.TaskMessage.OrchestrationInstance}', Action = '{nameof(SessionProvider)}.{nameof(AppendMessageAsync)}'"); + if (!added) + { + throw new InvalidOperationException($"No penidng or running orchestration found. Orchestration = '{newMessage.TaskMessage.OrchestrationInstance}', Action = 'AppendMessageAsync({newMessage.TaskMessage.Event.ToString()})'"); + } + this.TryEnqueueSession(newMessage.TaskMessage.OrchestrationInstance); } @@ -187,7 +199,7 @@ public async Task AppendMessageAsync(ITransaction transaction, TaskMessageItem n public async Task TryAppendMessageAsync(ITransaction transaction, TaskMessageItem newMessage) { - if (await this.Store.ContainsKeyAsync(transaction, newMessage.TaskMessage.OrchestrationInstance.InstanceId)) + if (await this.ContainsSessionAsync(transaction, newMessage.TaskMessage.OrchestrationInstance)) { var sessionMessageProvider = await GetOrAddSessionMessagesInstance(newMessage.TaskMessage.OrchestrationInstance); await sessionMessageProvider.SendBeginAsync(transaction, new Message(Guid.NewGuid(), newMessage)); @@ -197,6 +209,17 @@ public async Task TryAppendMessageAsync(ITransaction transaction, TaskMess return false; } + public async Task ContainsSessionAsync(ITransaction transaction, OrchestrationInstance instance) + { + var value = await this.Store.TryGetValueAsync(transaction, instance.InstanceId); + if (value.HasValue && OrchestrationInstanceComparer.Default.Equals(value.Value.SessionId, instance)) + { + return true; + } + + return false; + } + public async Task> TryAppendMessageBatchAsync(ITransaction transaction, IEnumerable newMessages) { List modifiedSessions = new List(); @@ -329,7 +352,7 @@ public void TryUnlockSession(OrchestrationInstance instance, bool abandon = fals } } - public async Task DropSession(ITransaction txn, OrchestrationInstance instance) + public async Task DropSessionAsync(ITransaction txn, OrchestrationInstance instance) { if (instance == null) { diff --git a/src/DurableTask.AzureServiceFabric/Utils.cs b/src/DurableTask.AzureServiceFabric/Utils.cs index 9309b0dab..692ecdbd8 100644 --- a/src/DurableTask.AzureServiceFabric/Utils.cs +++ b/src/DurableTask.AzureServiceFabric/Utils.cs @@ -23,25 +23,52 @@ namespace DurableTask.AzureServiceFabric internal static class Utils { - public static OrchestrationState BuildOrchestrationState(OrchestrationRuntimeState runtimeState) + public static OrchestrationState BuildOrchestrationState(TaskOrchestrationWorkItem workitem) { - return new OrchestrationState + var runtimeState = workitem.OrchestrationRuntimeState; + if (runtimeState == null) { - OrchestrationInstance = runtimeState.OrchestrationInstance, - ParentInstance = runtimeState.ParentInstance, - Name = runtimeState.Name, - Version = runtimeState.Version, - Status = runtimeState.Status, - Tags = runtimeState.Tags, - OrchestrationStatus = runtimeState.OrchestrationStatus, - CreatedTime = runtimeState.CreatedTime, - CompletedTime = runtimeState.CompletedTime, - LastUpdatedTime = DateTime.UtcNow, - Size = runtimeState.Size, - CompressedSize = runtimeState.CompressedSize, - Input = runtimeState.Input, - Output = runtimeState.Output - }; + throw new ArgumentNullException("runtimeState"); + } + + if (runtimeState.ExecutionStartedEvent != null) + { + return new OrchestrationState + { + OrchestrationInstance = runtimeState.OrchestrationInstance, + ParentInstance = runtimeState.ParentInstance, + Name = runtimeState.Name, + Version = runtimeState.Version, + Status = runtimeState.Status, + Tags = runtimeState.Tags, + OrchestrationStatus = runtimeState.OrchestrationStatus, + CreatedTime = runtimeState.CreatedTime, + CompletedTime = runtimeState.CompletedTime, + LastUpdatedTime = DateTime.UtcNow, + Size = runtimeState.Size, + CompressedSize = runtimeState.CompressedSize, + Input = runtimeState.Input, + Output = runtimeState.Output + }; + } + else + { + // Create a custom state for a bad orchestration + return new OrchestrationState + { + OrchestrationInstance = new OrchestrationInstance { InstanceId = workitem.InstanceId }, + ParentInstance = runtimeState.ParentInstance, + Status = runtimeState.Status, + Tags = runtimeState.Tags, + OrchestrationStatus = OrchestrationStatus.Terminated, + CompletedTime = runtimeState.CompletedTime, + LastUpdatedTime = DateTime.UtcNow, + Size = runtimeState.Size, + CompressedSize = runtimeState.CompressedSize, + Output = "Orchestration dropped" + }; + } + } public static TimeSpan Measure(Action action) diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs index ead1a233e..76d305d16 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs @@ -1054,8 +1054,7 @@ async Task AbandonAndReleaseSessionAsync(OrchestrationSession session) // will result in a duplicate replay of the orchestration with no side-effects. try { - session.ETag = await this.trackingStore.UpdateStateAsync(runtimeState, workItem.OrchestrationRuntimeState, instanceId, executionId, session.ETag); - + session.ETag = await this.trackingStore.UpdateStateAsync(runtimeState, workItem.OrchestrationRuntimeState, instanceId, executionId, session.ETag, session.TrackingStoreContext); // update the runtime state and execution id stored in the session session.UpdateRuntimeState(runtimeState); diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index f1e543783..ca969cdf7 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -225,6 +225,12 @@ public Task DownloadAndDecompressAsBytesAsync(Uri blobUri, CancellationT return DownloadAndDecompressAsBytesAsync(blob, cancellationToken); } + public Task DeleteBlobAsync(string blobName, CancellationToken cancellationToken = default) + { + Blob blob = this.blobContainer.GetBlobReference(blobName); + return blob.DeleteIfExistsAsync(cancellationToken); + } + private async Task DownloadAndDecompressAsBytesAsync(Blob blob, CancellationToken cancellationToken = default) { using (MemoryStream memory = new MemoryStream(MaxStorageQueuePayloadSizeInBytes * 2)) diff --git a/src/DurableTask.AzureStorage/Messaging/OrchestrationSession.cs b/src/DurableTask.AzureStorage/Messaging/OrchestrationSession.cs index 98acdf885..f11c82e8b 100644 --- a/src/DurableTask.AzureStorage/Messaging/OrchestrationSession.cs +++ b/src/DurableTask.AzureStorage/Messaging/OrchestrationSession.cs @@ -39,6 +39,7 @@ sealed class OrchestrationSession : SessionBase, IOrchestrationSession OrchestrationRuntimeState runtimeState, ETag? eTag, DateTime lastCheckpointTime, + object trackingStoreContext, TimeSpan idleTimeout, Guid traceActivityId) : base(settings, storageAccountName, orchestrationInstance, traceActivityId) @@ -49,6 +50,7 @@ sealed class OrchestrationSession : SessionBase, IOrchestrationSession this.RuntimeState = runtimeState ?? throw new ArgumentNullException(nameof(runtimeState)); this.ETag = eTag; this.LastCheckpointTime = lastCheckpointTime; + this.TrackingStoreContext = trackingStoreContext; this.messagesAvailableEvent = new AsyncAutoResetEvent(signaled: false); this.nextMessageBatch = new MessageCollection(); @@ -68,6 +70,8 @@ sealed class OrchestrationSession : SessionBase, IOrchestrationSession public DateTime LastCheckpointTime { get; } + public object TrackingStoreContext { get; } + public IReadOnlyList PendingMessages => this.nextMessageBatch; public override int GetCurrentEpisode() diff --git a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs index c2f6c726b..9cb7e4f15 100644 --- a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs +++ b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs @@ -479,6 +479,7 @@ bool IsScheduledAfterInstanceUpdate(MessageData msg, OrchestrationState? remoteI batch.OrchestrationState = new OrchestrationRuntimeState(history.Events); batch.ETag = history.ETag; batch.LastCheckpointTime = history.LastCheckpointTime; + batch.TrackingStoreContext = history.TrackingStoreContext; } this.readyForProcessingQueue.Enqueue(node); @@ -539,6 +540,7 @@ bool IsScheduledAfterInstanceUpdate(MessageData msg, OrchestrationState? remoteI nextBatch.OrchestrationState, nextBatch.ETag, nextBatch.LastCheckpointTime, + nextBatch.TrackingStoreContext, this.settings.ExtendedSessionIdleTimeout, traceActivityId); @@ -684,6 +686,7 @@ public PendingMessageBatch(ControlQueue controlQueue, string instanceId, string? public ETag? ETag { get; set; } public DateTime LastCheckpointTime { get; set; } + public object? TrackingStoreContext { get; set; } } } } diff --git a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs index af078f50b..da79a70d5 100644 --- a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs @@ -14,6 +14,7 @@ namespace DurableTask.AzureStorage.Tracking { using System; + using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -156,6 +157,7 @@ public override async Task GetHistoryEventsAsync(string in IList historyEvents; string executionId; TableEntity sentinel = null; + TrackingStoreContext trackingStoreContext = new TrackingStoreContext(); if (results.Entities.Count > 0) { // The most recent generation will always be in the first history event. @@ -181,7 +183,7 @@ public override async Task GetHistoryEventsAsync(string in } // Some entity properties may be stored in blob storage. - await this.DecompressLargeEntityProperties(entity, cancellationToken); + await this.DecompressLargeEntityProperties(entity, trackingStoreContext.Blobs, cancellationToken); events.Add((HistoryEvent)TableEntityConverter.Deserialize(entity, GetTypeForTableEntity(entity))); } @@ -221,7 +223,7 @@ public override async Task GetHistoryEventsAsync(string in eTagValue?.ToString(), checkpointCompletionTime); - return new OrchestrationHistory(historyEvents, checkpointCompletionTime, eTagValue); + return new OrchestrationHistory(historyEvents, checkpointCompletionTime, eTagValue, trackingStoreContext); } TableQueryResponse GetHistoryEntitiesResponseInfoAsync(string instanceId, string expectedExecutionId, IList projectionColumns, CancellationToken cancellationToken) @@ -717,7 +719,7 @@ public override async Task PurgeInstanceHistoryAsync(string // It is possible that the queue message was small enough to be written directly to a queue message, // not a blob, but is too large to be written to a table property. - await this.CompressLargeMessageAsync(entity, cancellationToken); + await this.CompressLargeMessageAsync(entity, listOfBlobs: null, cancellationToken: cancellationToken); Stopwatch stopwatch = Stopwatch.StartNew(); try @@ -803,11 +805,13 @@ public override Task StartAsync(CancellationToken cancellationToken = default) string instanceId, string executionId, ETag? eTagValue, + object trackingStoreContext, CancellationToken cancellationToken = default) { int estimatedBytes = 0; IList newEvents = newRuntimeState.NewEvents; IList allEvents = newRuntimeState.Events; + TrackingStoreContext context = (TrackingStoreContext) trackingStoreContext; int episodeNumber = Utils.GetEpisodeNumber(newRuntimeState); @@ -825,7 +829,15 @@ public override Task StartAsync(CancellationToken cancellationToken = default) ["ExecutionId"] = executionId, ["LastUpdatedTime"] = newEvents.Last().Timestamp, }; - + + // check if we are replacing a previous execution with blobs; those will be deleted from the store after the update. This could occur in a ContinueAsNew scenario + List blobsToDelete = null; + if (oldRuntimeState != newRuntimeState && context.Blobs.Count > 0) + { + blobsToDelete = context.Blobs; + context.Blobs = new List(); + } + for (int i = 0; i < newEvents.Count; i++) { bool isFinalEvent = i == newEvents.Count - 1; @@ -841,7 +853,7 @@ public override Task StartAsync(CancellationToken cancellationToken = default) historyEntity.RowKey = sequenceNumber.ToString("X16"); historyEntity["ExecutionId"] = executionId; - await this.CompressLargeMessageAsync(historyEntity, cancellationToken); + await this.CompressLargeMessageAsync(historyEntity, context.Blobs, cancellationToken); // Replacement can happen if the orchestration episode gets replayed due to a commit failure in one of the steps below. historyEventBatch.Add(new TableTransactionAction(TableTransactionActionType.UpsertReplace, historyEntity)); @@ -982,6 +994,18 @@ public override Task StartAsync(CancellationToken cancellationToken = default) episodeNumber, orchestrationInstanceUpdateStopwatch.ElapsedMilliseconds); + // finally, delete orphaned blobs from the previous execution history. + // We had to wait until the new history has committed to make sure the blobs are no longer necessary. + if (blobsToDelete != null) + { + var tasks = new List(blobsToDelete.Count); + foreach (var blobName in blobsToDelete) + { + tasks.Add(this.messageManager.DeleteBlobAsync(blobName)); + } + await Task.WhenAll(tasks); + } + return eTagValue; } @@ -1047,7 +1071,7 @@ Type GetTypeForTableEntity(TableEntity tableEntity) } } - async Task CompressLargeMessageAsync(TableEntity entity, CancellationToken cancellationToken) + async Task CompressLargeMessageAsync(TableEntity entity, List listOfBlobs, CancellationToken cancellationToken) { foreach (string propertyName in VariableSizeEntityProperties) { @@ -1065,11 +1089,14 @@ async Task CompressLargeMessageAsync(TableEntity entity, CancellationToken cance string blobPropertyName = GetBlobPropertyName(propertyName); entity.Add(blobPropertyName, blobName); entity[propertyName] = string.Empty; + + // if necessary, keep track of all the blobs associated with this execution + listOfBlobs?.Add(blobName); } } } - async Task DecompressLargeEntityProperties(TableEntity entity, CancellationToken cancellationToken) + async Task DecompressLargeEntityProperties(TableEntity entity, List listOfBlobs, CancellationToken cancellationToken) { // Check for entity properties stored in blob storage foreach (string propertyName in VariableSizeEntityProperties) @@ -1080,6 +1107,9 @@ async Task DecompressLargeEntityProperties(TableEntity entity, CancellationToken string decompressedMessage = await this.messageManager.DownloadAndDecompressAsBytesAsync(blobName, cancellationToken); entity[propertyName] = decompressedMessage; entity.Remove(blobPropertyName); + + // keep track of all the blobs associated with this execution + listOfBlobs.Add(blobName); } } } @@ -1111,7 +1141,10 @@ static string GetBlobName(TableEntity entity, string property) throw new InvalidOperationException($"Could not compute the blob name for property {property}"); } - return $"{sanitizedInstanceId}/history-{sequenceNumber}-{eventType}-{property}.json.gz"; + // randomize the blob name to prevent accidental races in split-brain situations (#890) + uint random = (uint)(new Random()).Next(); + + return $"{sanitizedInstanceId}/history-{sequenceNumber}-{eventType}-{random:X8}-{property}.json.gz"; } async Task UploadHistoryBatch( @@ -1210,5 +1243,10 @@ bool ExceedsMaxTablePropertySize(string data) return false; } + + class TrackingStoreContext + { + public List Blobs { get; set; } = new List(); + } } } diff --git a/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs index ca8d8de2e..db6338bfb 100644 --- a/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs @@ -73,8 +73,9 @@ interface ITrackingStore /// InstanceId for the Orchestration Update /// ExecutionId for the Orchestration Update /// The ETag value to use for safe updates + /// Additional context for the execution that is maintained by the tracking store. /// The token to monitor for cancellation requests. The default value is . - Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, ETag? eTag, CancellationToken cancellationToken = default); + Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, ETag? eTag, object trackingStoreContext, CancellationToken cancellationToken = default); /// /// Get The Orchestration State for the Latest or All Executions diff --git a/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs index 9a41d234e..4b816aacd 100644 --- a/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs @@ -136,7 +136,7 @@ public override Task StartAsync(CancellationToken cancellationToken = default) } /// - public override async Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, ETag? eTag, CancellationToken cancellationToken = default) + public override async Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, ETag? eTag, object executionData, CancellationToken cancellationToken = default) { //In case there is a runtime state for an older execution/iteration as well that needs to be committed, commit it. //This may be the case if a ContinueAsNew was executed on the orchestration diff --git a/src/DurableTask.AzureStorage/Tracking/OrchestrationHistory.cs b/src/DurableTask.AzureStorage/Tracking/OrchestrationHistory.cs index 436f9641b..94d66b779 100644 --- a/src/DurableTask.AzureStorage/Tracking/OrchestrationHistory.cs +++ b/src/DurableTask.AzureStorage/Tracking/OrchestrationHistory.cs @@ -31,10 +31,16 @@ public OrchestrationHistory(IList historyEvents, DateTime lastChec } public OrchestrationHistory(IList historyEvents, DateTime lastCheckpointTime, ETag? eTag) + : this(historyEvents, lastCheckpointTime, eTag, null) + { + } + + public OrchestrationHistory(IList historyEvents, DateTime lastCheckpointTime, ETag? eTag, object trackingStoreContext) { this.Events = historyEvents ?? throw new ArgumentNullException(nameof(historyEvents)); this.LastCheckpointTime = lastCheckpointTime; this.ETag = eTag; + this.TrackingStoreContext = trackingStoreContext; } public IList Events { get; } @@ -42,5 +48,7 @@ public OrchestrationHistory(IList historyEvents, DateTime lastChec public ETag? ETag { get; } public DateTime LastCheckpointTime { get; } + + public object TrackingStoreContext { get; } } } diff --git a/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs b/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs index 8a738a55a..95465461f 100644 --- a/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs +++ b/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs @@ -104,6 +104,6 @@ public virtual Task UpdateStatusForRewindAsync(string instanceId, CancellationTo public abstract Task StartAsync(CancellationToken cancellationToken = default); /// - public abstract Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, ETag? eTag, CancellationToken cancellationToken = default); + public abstract Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, ETag? eTag, object executionData, CancellationToken cancellationToken = default); } } diff --git a/test/DurableTask.AzureServiceFabric.Integration.Tests/FunctionalTests.cs b/test/DurableTask.AzureServiceFabric.Integration.Tests/FunctionalTests.cs index ce28f700e..7d43c18ab 100644 --- a/test/DurableTask.AzureServiceFabric.Integration.Tests/FunctionalTests.cs +++ b/test/DurableTask.AzureServiceFabric.Integration.Tests/FunctionalTests.cs @@ -414,7 +414,7 @@ public async Task ForceTerminate_Twice_Terminates_LatestExecution() var testData = new TestOrchestrationData() { NumberOfParallelTasks = 0, - NumberOfSerialTasks = 2, + NumberOfSerialTasks = 100, MaxDelay = 5, MinDelay = 5, DelayUnit = TimeSpan.FromSeconds(1), @@ -434,6 +434,115 @@ public async Task ForceTerminate_Twice_Terminates_LatestExecution() Assert.AreEqual(reason, result.Output); } + [TestMethod] + public async Task ForceTerminate_Already_Finished_Orchestration() + { + var testData = new TestOrchestrationData() + { + NumberOfParallelTasks = 0, + NumberOfSerialTasks = 2, + MaxDelay = 1, + MinDelay = 1, + DelayUnit = TimeSpan.FromMilliseconds(1), + }; + + var instance = await this.taskHubClient.CreateOrchestrationInstanceAsync(typeof(TestOrchestration), testData); + await Task.Delay(TimeSpan.FromSeconds(1)); + + var result = await this.taskHubClient.WaitForOrchestrationAsync(instance, TimeSpan.FromMinutes(1)); + + Assert.AreEqual(OrchestrationStatus.Completed, result.OrchestrationStatus); + + var reason = "Testing terminatiom of already finished orchestration"; + + await Assert.ThrowsExceptionAsync(() => this.taskHubClient.TerminateInstanceAsync(instance, reason)); + } + + [TestMethod] + public async Task ForceTerminate_With_CleanupStore() + { + var testData = new TestOrchestrationData() + { + NumberOfParallelTasks = 0, + NumberOfSerialTasks = 5, + MaxDelay = 5, + MinDelay = 5, + DelayUnit = TimeSpan.FromSeconds(1), + }; + + var instance = await this.taskHubClient.CreateOrchestrationInstanceAsync(typeof(TestOrchestration), testData); + await Task.Delay(TimeSpan.FromMilliseconds(1)); + + var reason = "CleanupStore"; + await this.taskHubClient.TerminateInstanceAsync(instance, reason); + var result = await this.taskHubClient.WaitForOrchestrationAsync(instance, TimeSpan.FromMinutes(1)); + + Assert.AreEqual(OrchestrationStatus.Terminated, result.OrchestrationStatus); + Assert.IsTrue(result.Output.Contains(reason)); + } + + [TestMethod] + public async Task ForceTerminate_With_CleanupStore_Start_New_Orchestration_With_Same_InstanceId() + { + string instanceId = "ForceTerminate_With_CleanupStore_Start_New_Orchestration_With_Same_InstanceId"; + var testData = new TestOrchestrationData() + { + NumberOfParallelTasks = 0, + NumberOfSerialTasks = 5, + MaxDelay = 5, + MinDelay = 5, + DelayUnit = TimeSpan.FromSeconds(1), + }; + + var instance = await this.taskHubClient.CreateOrchestrationInstanceAsync(typeof(TestOrchestration), instanceId, testData); + await Task.Delay(TimeSpan.FromMilliseconds(1)); + + var reason = "CleanupStore"; + await this.taskHubClient.TerminateInstanceAsync(instance, reason); + var result = await this.taskHubClient.WaitForOrchestrationAsync(instance, TimeSpan.FromMinutes(1)); + + Assert.AreEqual(OrchestrationStatus.Terminated, result.OrchestrationStatus); + Assert.IsTrue(result.Output.Contains(reason)); + + instance = await this.taskHubClient.CreateOrchestrationInstanceAsync(typeof(TestOrchestration), instanceId, testData); + await Task.Delay(TimeSpan.FromMilliseconds(1)); + + result = await this.taskHubClient.WaitForOrchestrationAsync(instance, TimeSpan.FromMinutes(1)); + + Assert.AreEqual(OrchestrationStatus.Completed, result.OrchestrationStatus); + } + + [TestMethod] + public async Task ForceTerminate_And_Start_New_Orchestration_With_Same_InstanceId() + { + string instanceId = "ForceTerminate_And_Start_New_Orchestration_With_Same_InstanceId"; + var testData = new TestOrchestrationData() + { + NumberOfParallelTasks = 0, + NumberOfSerialTasks = 5, + MaxDelay = 5, + MinDelay = 5, + DelayUnit = TimeSpan.FromSeconds(1), + }; + + var instance = await this.taskHubClient.CreateOrchestrationInstanceAsync(typeof(TestOrchestration), instanceId, testData); + await Task.Delay(TimeSpan.FromMilliseconds(1)); + + var reason = "ForceTerminate_And_Start_New_Orchestration_With_Same_InstanceId"; + await this.taskHubClient.TerminateInstanceAsync(instance, reason); + var result = await this.taskHubClient.WaitForOrchestrationAsync(instance, TimeSpan.FromMinutes(1)); + + Assert.AreEqual(OrchestrationStatus.Terminated, result.OrchestrationStatus); + Assert.IsTrue(result.Output.Contains(reason)); + + instance = await this.taskHubClient.CreateOrchestrationInstanceAsync(typeof(TestOrchestration), instanceId, testData); + await Task.Delay(TimeSpan.FromMilliseconds(1)); + + result = await this.taskHubClient.WaitForOrchestrationAsync(instance, TimeSpan.FromMinutes(1)); + + Assert.AreEqual(OrchestrationStatus.Completed, result.OrchestrationStatus); + } + [TestMethod] public async Task Purge_Removes_State() { From 64d381c4e72e339ba1ec05cfcf377ff04c741763 Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Fri, 28 Jul 2023 12:35:47 -0700 Subject: [PATCH 08/62] Downgrade version --- src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 5d824dbce..7acdc2ba7 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -36,12 +36,12 @@ - + - + From 721a85a3be7ba397c6c81db0dbf818cc90b7f51b Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Fri, 28 Jul 2023 12:42:16 -0700 Subject: [PATCH 09/62] Nitpick --- src/DurableTask.AzureStorage/AssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/AssemblyInfo.cs b/src/DurableTask.AzureStorage/AssemblyInfo.cs index f817ed5c6..15255eaa0 100644 --- a/src/DurableTask.AzureStorage/AssemblyInfo.cs +++ b/src/DurableTask.AzureStorage/AssemblyInfo.cs @@ -16,4 +16,4 @@ #if !SIGN_ASSEMBLY [assembly: InternalsVisibleTo("DurableTask.AzureStorage.Tests")] [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] -#endif +#endif \ No newline at end of file From 356acce48a66208e52744a51f8b4be280ca6c51d Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Mon, 31 Jul 2023 17:06:50 -0700 Subject: [PATCH 10/62] Update version suffix --- src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 7acdc2ba7..6c3e2187a 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -24,7 +24,7 @@ 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) - preview.2 + preview.3 $(VersionPrefix).0 $(VersionPrefix).$(FileVersionRevision) From 500ab129a72e8823ff253ce707f5adee2fb300e6 Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Wed, 16 Aug 2023 14:34:06 -0700 Subject: [PATCH 11/62] Update Azure Storage SDKs (#940) * SF Provider: memory leak: Create a new scope in HTTP controller DI on each BeginScope call (#933) * SF Provider: memory leak: Create a new scope in HTTP controller DI on every BeginScope call * Bump up package version --------- Co-authored-by: Abhineet Garg * Update SDK versions * Use Uri property * Update table sdk again --------- Co-authored-by: NeetArt <20233106+NeetArt@users.noreply.github.com> Co-authored-by: Abhineet Garg --- .../Storage/TableClientExtensionsTests.cs | 57 ------------------- .../DurableTask.AzureServiceFabric.csproj | 2 +- .../Service/DefaultDependencyResolver.cs | 17 +++++- .../DurableTask.AzureStorage.csproj | 14 ++--- src/DurableTask.AzureStorage/Storage/Table.cs | 2 +- .../Storage/TableClientExtensions.cs | 51 ----------------- .../Tracking/AzureTableTrackingStore.cs | 7 +-- 7 files changed, 23 insertions(+), 127 deletions(-) delete mode 100644 Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs delete mode 100644 src/DurableTask.AzureStorage/Storage/TableClientExtensions.cs diff --git a/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs b/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs deleted file mode 100644 index 31bdd4d85..000000000 --- a/Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs +++ /dev/null @@ -1,57 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- - -namespace DurableTask.AzureStorage.Tests.Storage -{ - using System; - using Azure.Data.Tables; - using DurableTask.AzureStorage.Storage; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class TableClientExtensionsTests - { - const string EmulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; - - [TestMethod] - public void GetUri_ConnectionString() - { - TableClient client; - - client = new TableClient("UseDevelopmentStorage=true", "bar"); - Assert.AreEqual(new Uri("http://127.0.0.1:10002/devstoreaccount1/bar"), client.GetUri()); - - client = new TableClient($"DefaultEndpointsProtocol=https;AccountName=foo;AccountKey={EmulatorAccountKey};TableEndpoint=https://foo.table.core.windows.net/;", "bar"); - Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri()); - } - - [TestMethod] - public void GetUri_ServiceEndpoint() - { - var client = new TableClient(new Uri("https://foo.table.core.windows.net/"), "bar", new TableSharedKeyCredential("foo", EmulatorAccountKey)); - Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri()); - } - - [TestMethod] - public void GetUri_TableEndpoint() - { - TableClient client; - - client = new TableClient(new Uri("https://foo.table.core.windows.net/bar")); - Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri()); - - client = new TableClient(new Uri("https://foo.table.core.windows.net/bar?sv=2019-12-12&ss=t&srt=s&sp=rwdlacu&se=2020-08-28T23:45:30Z&st=2020-08-26T15:45:30Z&spr=https&sig=mySig&tn=bar")); - Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri()); - } - } -} \ No newline at end of file diff --git a/src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj b/src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj index 5666c8b87..900102a4f 100644 --- a/src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj +++ b/src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj @@ -6,7 +6,7 @@ true Microsoft.Azure.DurableTask.AzureServiceFabric true - 2.3.9 + 2.3.10 $(Version) $(Version) Azure Service Fabric provider extension for the Durable Task Framework. diff --git a/src/DurableTask.AzureServiceFabric/Service/DefaultDependencyResolver.cs b/src/DurableTask.AzureServiceFabric/Service/DefaultDependencyResolver.cs index b90c7fbca..3361d009e 100644 --- a/src/DurableTask.AzureServiceFabric/Service/DefaultDependencyResolver.cs +++ b/src/DurableTask.AzureServiceFabric/Service/DefaultDependencyResolver.cs @@ -22,7 +22,8 @@ namespace DurableTask.AzureServiceFabric.Service /// public sealed class DefaultDependencyResolver : IDependencyResolver { - private IServiceProvider provider; + private readonly IServiceProvider provider; + private readonly IServiceScope scope; /// /// Creates an instance of . @@ -33,6 +34,16 @@ public DefaultDependencyResolver(IServiceProvider provider) this.provider = provider ?? throw new ArgumentNullException(nameof(provider)); } + /// + /// Creates a private instance of used when creating a new scope. + /// + /// An instance of + private DefaultDependencyResolver(IServiceScope scope) + { + this.scope = scope; + this.provider = scope.ServiceProvider; + } + /// public object GetService(Type serviceType) { @@ -48,14 +59,14 @@ public IEnumerable GetServices(Type serviceType) /// public IDependencyScope BeginScope() { - return this; + return new DefaultDependencyResolver(this.provider.CreateScope()); } #region IDisposable Support /// public void Dispose() { - // no-op + this.scope?.Dispose(); } #endregion } diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 6c3e2187a..d0eda7cad 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -24,7 +24,7 @@ 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) - preview.3 + preview.4 $(VersionPrefix).0 $(VersionPrefix).$(FileVersionRevision) @@ -34,15 +34,11 @@ $(VersionPrefix)-$(VersionSuffix) - - - - - - - - + + + + diff --git a/src/DurableTask.AzureStorage/Storage/Table.cs b/src/DurableTask.AzureStorage/Storage/Table.cs index 8983f3e04..11e6170f7 100644 --- a/src/DurableTask.AzureStorage/Storage/Table.cs +++ b/src/DurableTask.AzureStorage/Storage/Table.cs @@ -42,7 +42,7 @@ public Table(AzureStorageClient azureStorageClient, TableServiceClient tableServ public string Name => this.tableClient.Name; - internal Uri? Uri => this.tableClient.GetUri(); // TODO: Replace with Uri property + internal Uri Uri => this.tableClient.Uri; public async Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default) { diff --git a/src/DurableTask.AzureStorage/Storage/TableClientExtensions.cs b/src/DurableTask.AzureStorage/Storage/TableClientExtensions.cs deleted file mode 100644 index da2b73b1f..000000000 --- a/src/DurableTask.AzureStorage/Storage/TableClientExtensions.cs +++ /dev/null @@ -1,51 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- -#nullable enable -namespace DurableTask.AzureStorage.Storage -{ - using System; - using System.Linq.Expressions; - using System.Reflection; - using Azure.Data.Tables; - - static class TableClientExtensions - { - public static Uri? GetUri(this TableClient tableClient) - { - if (tableClient == null) - { - throw new ArgumentNullException(nameof(tableClient)); - } - - Uri? endpoint = GetEndpointFunc(tableClient); - return endpoint != null - ? new TableUriBuilder(endpoint) { Query = null, Sas = null, Tablename = tableClient.Name }.ToUri() - : null; - } - - static readonly Func GetEndpointFunc = CreateGetEndpointFunc(); - - static Func CreateGetEndpointFunc() - { - Type tableClientType = typeof(TableClient); - ParameterExpression clientParam = Expression.Parameter(typeof(TableClient), "client"); - FieldInfo? endpointField = tableClientType.GetField("_endpoint", BindingFlags.Instance | BindingFlags.NonPublic); - - Expression> lambdaExpr = endpointField != null - ? Expression.Lambda>(Expression.Field(clientParam, endpointField), clientParam) - : Expression.Lambda>(Expression.Constant(null, typeof(Uri)), clientParam); - - return lambdaExpr.Compile(); - } - } -} diff --git a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs index 2c559b547..39bbda7c0 100644 --- a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs @@ -789,11 +789,8 @@ public override async Task UpdateStatusForRewindAsync(string instanceId, Cancell /// public override Task StartAsync(CancellationToken cancellationToken = default) { - if (this.HistoryTable.Uri != null) - ServicePointManager.FindServicePoint(this.HistoryTable.Uri).UseNagleAlgorithm = false; - - if (this.InstancesTable.Uri != null) - ServicePointManager.FindServicePoint(this.InstancesTable.Uri).UseNagleAlgorithm = false; + ServicePointManager.FindServicePoint(this.HistoryTable.Uri).UseNagleAlgorithm = false; + ServicePointManager.FindServicePoint(this.InstancesTable.Uri).UseNagleAlgorithm = false; return Task.CompletedTask; } From 572d2ef82a45503e1292bacf58dfbe888e7ba38a Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Sat, 26 Aug 2023 18:04:06 -0700 Subject: [PATCH 12/62] Remove SimpleBufferManager --- .../MessageManager.cs | 83 ++----- .../Partitioning/BlobPartitionLeaseManager.cs | 21 +- .../SimpleBufferManager.cs | 139 ------------ src/DurableTask.AzureStorage/Storage/Blob.cs | 20 +- src/DurableTask.AzureStorage/Utils.cs | 28 +++ ...zureServiceFabric.Integration.Tests.csproj | 6 +- ...urableTask.AzureServiceFabric.Tests.csproj | 6 +- .../BufferManagerTests.cs | 206 ------------------ .../DurableTask.AzureStorage.Tests.csproj | 6 +- .../DurableTask.Core.Tests.csproj | 6 +- .../DurableTask.Emulator.Tests.csproj | 6 +- .../DurableTask.Redis.Tests.csproj | 2 +- .../DurableTask.ServiceBus.Tests.csproj | 6 +- .../DurableTask.SqlServer.Tests.csproj | 6 +- 14 files changed, 92 insertions(+), 449 deletions(-) delete mode 100644 src/DurableTask.AzureStorage/SimpleBufferManager.cs delete mode 100644 test/DurableTask.AzureStorage.Tests/BufferManagerTests.cs diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index ca969cdf7..6362d2d2f 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -25,6 +25,7 @@ namespace DurableTask.AzureStorage using System.Threading; using System.Threading.Tasks; using Azure; + using Azure.Storage.Blobs.Models; using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Storage; using Newtonsoft.Json; @@ -175,39 +176,25 @@ internal MessageData DeserializeMessageData(string json) return Utils.DeserializeFromJson(this.serializer, json); } - public Task CompressAndUploadAsBytesAsync(byte[] payloadBuffer, string blobName, CancellationToken cancellationToken = default) + public async Task CompressAndUploadAsBytesAsync(byte[] payloadBuffer, string blobName, CancellationToken cancellationToken = default) { - ArraySegment compressedSegment = this.Compress(payloadBuffer); - return this.UploadToBlobAsync(compressedSegment.Array, compressedSegment.Count, blobName, cancellationToken); - } + await this.EnsureContainerAsync(cancellationToken); - public ArraySegment Compress(byte[] payloadBuffer) - { - using (var originStream = new MemoryStream(payloadBuffer, 0, payloadBuffer.Length)) + Blob blob = this.blobContainer.GetBlobReference(blobName); + using Stream blobStream = await blob.OpenWriteAsync(cancellationToken); + using GZipStream compressedStream = new GZipStream(blobStream, CompressionLevel.Optimal); + using MemoryStream payloadStream = new MemoryStream(payloadBuffer); + + try { - using (MemoryStream memory = new MemoryStream()) - { - using (GZipStream gZipStream = new GZipStream(memory, CompressionLevel.Optimal, leaveOpen: true)) - { - byte[] buffer = SimpleBufferManager.Shared.TakeBuffer(DefaultBufferSize); - try - { - int read; - while ((read = originStream.Read(buffer, 0, DefaultBufferSize)) != 0) - { - gZipStream.Write(buffer, 0, read); - } - - gZipStream.Flush(); - } - finally - { - SimpleBufferManager.Shared.ReturnBuffer(buffer); - } - } - - return new ArraySegment(memory.GetBuffer(), 0, (int)memory.Length); - } + // TODO: Change default pipe options if necessary + await payloadStream.CopyToAsync(compressedStream, bufferSize: 81920, cancellationToken: cancellationToken); + await compressedStream.FlushAsync(cancellationToken); + await blobStream.FlushAsync(cancellationToken); + } + catch (RequestFailedException rfe) + { + throw new DurableTaskStorageException(rfe); } } @@ -233,14 +220,11 @@ public Task DeleteBlobAsync(string blobName, CancellationToken cancellatio private async Task DownloadAndDecompressAsBytesAsync(Blob blob, CancellationToken cancellationToken = default) { - using (MemoryStream memory = new MemoryStream(MaxStorageQueuePayloadSizeInBytes * 2)) - { - await blob.DownloadToStreamAsync(memory, cancellationToken); - memory.Position = 0; + using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken); + using GZipStream decompressedStream = new GZipStream(result.Content, CompressionMode.Decompress); + using StreamReader reader = new StreamReader(decompressedStream, Encoding.UTF8); - ArraySegment decompressedSegment = this.Decompress(memory); - return Encoding.UTF8.GetString(decompressedSegment.Array, 0, decompressedSegment.Count); - } + return await reader.ReadToEndAsync(); } public string GetBlobUrl(string blobName) @@ -248,31 +232,6 @@ public string GetBlobUrl(string blobName) return Uri.UnescapeDataString(this.blobContainer.GetBlobReference(blobName).Uri.AbsoluteUri); } - public ArraySegment Decompress(Stream blobStream) - { - using (GZipStream gZipStream = new GZipStream(blobStream, CompressionMode.Decompress)) - { - using (MemoryStream memory = new MemoryStream(MaxStorageQueuePayloadSizeInBytes * 2)) - { - byte[] buffer = SimpleBufferManager.Shared.TakeBuffer(DefaultBufferSize); - try - { - int count = 0; - while ((count = gZipStream.Read(buffer, 0, DefaultBufferSize)) > 0) - { - memory.Write(buffer, 0, count); - } - } - finally - { - SimpleBufferManager.Shared.ReturnBuffer(buffer); - } - - return new ArraySegment(memory.GetBuffer(), 0, (int)memory.Length); - } - } - } - public MessageFormatFlags GetMessageFormatFlags(MessageData messageData) { MessageFormatFlags messageFormatFlags = MessageFormatFlags.InlineJson; diff --git a/src/DurableTask.AzureStorage/Partitioning/BlobPartitionLeaseManager.cs b/src/DurableTask.AzureStorage/Partitioning/BlobPartitionLeaseManager.cs index 07335ab06..932f5bd7f 100644 --- a/src/DurableTask.AzureStorage/Partitioning/BlobPartitionLeaseManager.cs +++ b/src/DurableTask.AzureStorage/Partitioning/BlobPartitionLeaseManager.cs @@ -16,14 +16,13 @@ namespace DurableTask.AzureStorage.Partitioning using System; using System.Collections.Generic; using System.Globalization; - using System.IO; using System.Linq; using System.Net; using System.Runtime.CompilerServices; - using System.Text; using System.Threading; using System.Threading.Tasks; using Azure; + using Azure.Storage.Blobs.Models; using DurableTask.AzureStorage.Storage; sealed class BlobPartitionLeaseManager : ILeaseManager @@ -318,22 +317,8 @@ async Task GetTaskHubInfoAsync(CancellationToken cancellationToken) async Task DownloadLeaseBlob(Blob blob, CancellationToken cancellationToken) { - string serializedLease = null; - var buffer = SimpleBufferManager.Shared.TakeBuffer(SimpleBufferManager.SmallBufferSize); - try - { - using (var memoryStream = new MemoryStream(buffer)) - { - await blob.DownloadToStreamAsync(memoryStream, cancellationToken); - serializedLease = Encoding.UTF8.GetString(buffer, 0, (int)memoryStream.Position); - } - } - finally - { - SimpleBufferManager.Shared.ReturnBuffer(buffer); - } - - BlobPartitionLease deserializedLease = Utils.DeserializeFromJson(serializedLease); + using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken); + BlobPartitionLease deserializedLease = Utils.DeserializeFromJson(result.Content); deserializedLease.Blob = blob; return deserializedLease; diff --git a/src/DurableTask.AzureStorage/SimpleBufferManager.cs b/src/DurableTask.AzureStorage/SimpleBufferManager.cs deleted file mode 100644 index 44e64bec7..000000000 --- a/src/DurableTask.AzureStorage/SimpleBufferManager.cs +++ /dev/null @@ -1,139 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- - -namespace DurableTask.AzureStorage -{ - using System; - using System.Collections.Concurrent; - using System.Threading; - - /// - /// Simple buffer manager intended for use with Azure Storage SDK and compression code. - /// It is not intended to be robust enough for external use. - /// - class SimpleBufferManager - { - internal const int MaxBufferSize = 1024 * 1024; // 1 MB - const int DefaultBufferSize = 64 * 1024; // 64 KB - public const int SmallBufferSize = 1024; // 1 KB - - /// - /// Shared singleton instance of . - /// - public static SimpleBufferManager Shared { get; } = new SimpleBufferManager(); - - /// - /// Internal pool of buffers. Using stacks internally ensures that the same pool can be - /// frequently reused, which can result in improved performance due to hardware caching. - /// - readonly ConcurrentDictionary> pool; - - int allocatedBytes; - int availableBytes; - - /// - /// Initializes a new instance of the class. - /// - public SimpleBufferManager() - { - this.pool = new ConcurrentDictionary>(); - } - - /// - /// The total number of bytes allocated by this buffer. - /// - public int AllocatedBytes => this.allocatedBytes; - - /// - /// The total bytes available to be reused. - /// - public int AvailableBytes => this.availableBytes; - - /// - /// The number of buckets allocated by this pool. - /// - public int BucketCount => this.pool.Count; - - /// - public int GetDefaultBufferSize() - { - return DefaultBufferSize; - } - - /// - public void ReturnBuffer(byte[] buffer) - { - if (buffer == null) - { - throw new ArgumentNullException(nameof(buffer)); - } - - int bufferSize = buffer.Length; - if (bufferSize > MaxBufferSize) - { - // This was a large buffer which we're not tracking. - return; - } - - ConcurrentStack bucket; - if (!this.pool.TryGetValue(bufferSize, out bucket)) - { - throw new ArgumentException("The returned buffer did not come from this pool.", nameof(buffer)); - } - - bucket.Push(buffer); - Interlocked.Add(ref this.availableBytes, bufferSize); - } - - /// - public byte[] TakeBuffer(int bufferSize) - { - if (bufferSize > MaxBufferSize) - { - // We don't track large buffers - return new byte[bufferSize]; - } - - bufferSize = (bufferSize < 0) ? DefaultBufferSize : bufferSize; - - ConcurrentStack bucket = this.pool.GetOrAdd( - bufferSize, - size => new ConcurrentStack()); - - byte[] buffer; - if (bucket.TryPop(out buffer)) - { - Interlocked.Add(ref this.availableBytes, -bufferSize); - } - else - { - buffer = new byte[bufferSize]; - Interlocked.Add(ref this.allocatedBytes, bufferSize); - } - - return buffer; - } - - /// - /// Returns a debug string representing the current state of the buffer manager. - /// - public override string ToString() - { - return string.Format( - "BucketCount: {0}, AvailableBytes: {1}, AllocatedBytes: {2}.", - this.BucketCount, - this.AvailableBytes, - this.AllocatedBytes); - } - } -} diff --git a/src/DurableTask.AzureStorage/Storage/Blob.cs b/src/DurableTask.AzureStorage/Storage/Blob.cs index 22b92116f..390a92607 100644 --- a/src/DurableTask.AzureStorage/Storage/Blob.cs +++ b/src/DurableTask.AzureStorage/Storage/Blob.cs @@ -15,7 +15,6 @@ namespace DurableTask.AzureStorage.Storage { using System; using System.IO; - using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -90,9 +89,21 @@ public async Task UploadFromByteArrayAsync(byte[] buffer, int index, int byteCou await this.blockBlobClient.UploadAsync(stream, cancellationToken: cancellationToken).DecorateFailure(); } + public async Task OpenWriteAsync(CancellationToken cancellationToken = default) + { + try + { + return await this.blockBlobClient.OpenWriteAsync(overwrite: true, cancellationToken: cancellationToken); + } + catch (RequestFailedException rfe) + { + throw new DurableTaskStorageException(rfe); + } + } + public async Task DownloadTextAsync(CancellationToken cancellationToken = default) { - BlobDownloadStreamingResult result = await this.blockBlobClient.DownloadStreamingAsync(cancellationToken: cancellationToken).DecorateFailure(); + using BlobDownloadStreamingResult result = await this.blockBlobClient.DownloadStreamingAsync(cancellationToken: cancellationToken).DecorateFailure(); using var reader = new StreamReader(result.Content, Encoding.UTF8); return await reader.ReadToEndAsync(); @@ -103,6 +114,11 @@ public Task DownloadToStreamAsync(MemoryStream target, CancellationToken cancell return this.blockBlobClient.DownloadToAsync(target, cancellationToken: cancellationToken).DecorateFailure(); } + public async Task DownloadStreamingAsync(CancellationToken cancellationToken = default) + { + return await this.blockBlobClient.DownloadStreamingAsync(cancellationToken: cancellationToken).DecorateFailure(); + } + public async Task AcquireLeaseAsync(TimeSpan leaseInterval, string leaseId, CancellationToken cancellationToken = default) { BlobLease lease = await this.blockBlobClient diff --git a/src/DurableTask.AzureStorage/Utils.cs b/src/DurableTask.AzureStorage/Utils.cs index c784f2a06..7db878ffc 100644 --- a/src/DurableTask.AzureStorage/Utils.cs +++ b/src/DurableTask.AzureStorage/Utils.cs @@ -202,6 +202,34 @@ public static T DeserializeFromJson(JsonSerializer serializer, string jsonStr return obj; } + /// + /// Deserialize a JSON-string into an object of type T + /// This utility is resilient to end-user changes in the DefaultSettings of Newtonsoft. + /// + /// The type to deserialize the JSON string into. + /// A stream of UTF-8 JSON. + /// The deserialized value. + public static T DeserializeFromJson(Stream stream) + { + return DeserializeFromJson(DefaultJsonSerializer, stream); + } + + /// + /// Deserialize a JSON-string into an object of type T + /// This utility is resilient to end-user changes in the DefaultSettings of Newtonsoft. + /// + /// The type to deserialize the JSON string into. + /// The serializer whose config will guide the deserialization. + /// A stream of UTF-8 JSON. + /// The deserialized value. + public static T DeserializeFromJson(JsonSerializer serializer, Stream stream) + { + using var reader = new StreamReader(stream, Encoding.UTF8); + using var jsonReader = new JsonTextReader(reader); + + return serializer.Deserialize(jsonReader); + } + /// /// Deserialize a JSON-string into an object of type T /// This utility is resilient to end-user changes in the DefaultSettings of Newtonsoft. diff --git a/test/DurableTask.AzureServiceFabric.Integration.Tests/DurableTask.AzureServiceFabric.Integration.Tests.csproj b/test/DurableTask.AzureServiceFabric.Integration.Tests/DurableTask.AzureServiceFabric.Integration.Tests.csproj index 36f4afdbb..a8198945e 100644 --- a/test/DurableTask.AzureServiceFabric.Integration.Tests/DurableTask.AzureServiceFabric.Integration.Tests.csproj +++ b/test/DurableTask.AzureServiceFabric.Integration.Tests/DurableTask.AzureServiceFabric.Integration.Tests.csproj @@ -8,9 +8,9 @@ - - - + + + diff --git a/test/DurableTask.AzureServiceFabric.Tests/DurableTask.AzureServiceFabric.Tests.csproj b/test/DurableTask.AzureServiceFabric.Tests/DurableTask.AzureServiceFabric.Tests.csproj index 56c98730c..ee2a5ceb0 100644 --- a/test/DurableTask.AzureServiceFabric.Tests/DurableTask.AzureServiceFabric.Tests.csproj +++ b/test/DurableTask.AzureServiceFabric.Tests/DurableTask.AzureServiceFabric.Tests.csproj @@ -7,9 +7,9 @@ - - - + + + diff --git a/test/DurableTask.AzureStorage.Tests/BufferManagerTests.cs b/test/DurableTask.AzureStorage.Tests/BufferManagerTests.cs deleted file mode 100644 index dea255283..000000000 --- a/test/DurableTask.AzureStorage.Tests/BufferManagerTests.cs +++ /dev/null @@ -1,206 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- - -namespace DurableTask.AzureStorage.Tests -{ - using System.Collections.Generic; - using System.Diagnostics; - using System.Threading.Tasks; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class BufferManagerTests - { - /// - /// Validates basic take and return operations. - /// - [TestMethod] - public void TakeAndReturnBuffer() - { - var bufferManager = new SimpleBufferManager(); - Trace.TraceInformation(bufferManager.ToString()); - Assert.AreEqual(0, bufferManager.BucketCount); - Assert.AreEqual(0, bufferManager.AllocatedBytes); - Assert.AreEqual(0, bufferManager.AvailableBytes); - - byte[] buffer1024 = bufferManager.TakeBuffer(1024); - Trace.TraceInformation(bufferManager.ToString()); - Assert.AreEqual(1024, buffer1024.Length); - Assert.AreEqual(1024, bufferManager.AllocatedBytes); - Assert.AreEqual(1, bufferManager.BucketCount); - Assert.AreEqual(0, bufferManager.AvailableBytes); - - bufferManager.ReturnBuffer(buffer1024); - Trace.TraceInformation(bufferManager.ToString()); - Assert.AreEqual(1024, bufferManager.AllocatedBytes); - Assert.AreEqual(1024, bufferManager.AvailableBytes); - Assert.AreEqual(1, bufferManager.BucketCount); - - byte[] buffer4096 = bufferManager.TakeBuffer(4096); - Trace.TraceInformation(bufferManager.ToString()); - Assert.AreEqual(4096, buffer4096.Length); - Assert.AreEqual(1024 + 4096, bufferManager.AllocatedBytes); - Assert.AreEqual(1024, bufferManager.AvailableBytes); - Assert.AreEqual(2, bufferManager.BucketCount); - - bufferManager.ReturnBuffer(buffer4096); - Trace.TraceInformation(bufferManager.ToString()); - Assert.AreEqual(1024 + 4096, bufferManager.AllocatedBytes); - Assert.AreEqual(1024 + 4096, bufferManager.AvailableBytes); - Assert.AreEqual(2, bufferManager.BucketCount); - } - - /// - /// Tests that buffers of the same size are correctly reused. - /// - [TestMethod] - public void RecycleBuffer() - { - var bufferManager = new SimpleBufferManager(); - int bufferSize = bufferManager.GetDefaultBufferSize(); - - for (int i = 0; i < 10; i++) - { - byte[] buffer = bufferManager.TakeBuffer(bufferSize); - Trace.TraceInformation(bufferManager.ToString()); - Assert.AreEqual(bufferSize, buffer.Length); - Assert.AreEqual(bufferSize, bufferManager.AllocatedBytes); - Assert.AreEqual(0, bufferManager.AvailableBytes); - Assert.AreEqual(1, bufferManager.BucketCount); - - bufferManager.ReturnBuffer(buffer); - Trace.TraceInformation(bufferManager.ToString()); - Assert.AreEqual(bufferSize, bufferManager.AllocatedBytes); - Assert.AreEqual(bufferSize, bufferManager.AvailableBytes); - Assert.AreEqual(1, bufferManager.BucketCount); - } - } - - /// - /// Tests concurrent buffer allocation and deallocation. - /// - [TestMethod] - public void ConcurrentAllocations() - { - var bufferManager = new SimpleBufferManager(); - int bufferSize = bufferManager.GetDefaultBufferSize(); - - const int ConcurrentThreads = 16; - var options = new ParallelOptions - { - MaxDegreeOfParallelism = ConcurrentThreads, - }; - - // Repeat the core test multiple times to ensure the exact same - // results each time (verifies the recycling behavior). - for (int i = 0; i < 5; i++) - { - var uniqueBuffers = new HashSet(); - Parallel.For(0, ConcurrentThreads, options, j => - { - byte[] buffer = bufferManager.TakeBuffer(bufferSize); - Assert.AreEqual(bufferSize, buffer.Length); - lock (uniqueBuffers) - { - Assert.AreEqual(true, uniqueBuffers.Add(buffer)); - } - }); - - Trace.TraceInformation($"Round {i} after allocation: {bufferManager}"); - Assert.AreEqual(ConcurrentThreads * bufferSize, bufferManager.AllocatedBytes); - Assert.AreEqual(ConcurrentThreads, uniqueBuffers.Count); - Assert.AreEqual(0, bufferManager.AvailableBytes); - Assert.AreEqual(1, bufferManager.BucketCount); - - Parallel.ForEach(uniqueBuffers, options, buffer => - { - bufferManager.ReturnBuffer(buffer); - }); - - Trace.TraceInformation($"Round {i} after deallocation: {bufferManager}"); - Assert.AreEqual(ConcurrentThreads * bufferSize, bufferManager.AvailableBytes); - Assert.AreEqual(bufferManager.AvailableBytes, bufferManager.AllocatedBytes); - Assert.AreEqual(1, bufferManager.BucketCount); - } - } - - /// - /// Verifies that the buffer manager can manage multiple buckets of different sizes. - /// - [TestMethod] - public void MultipleBuckets() - { - var bufferManager = new SimpleBufferManager(); - - var buffers = new HashSet(); - - int totalBytes = 0; - int bucketCount; - for (bucketCount = 1; bucketCount <= 3; bucketCount++) - { - int bufferSize = 1024 << (bucketCount - 1); - for (int i = 1; i <= 4; i++) - { - byte[] buffer = bufferManager.TakeBuffer(bufferSize); - Assert.AreEqual(bufferSize, buffer.Length); - Assert.IsTrue(buffers.Add(buffer)); - - totalBytes += bufferSize; - - Trace.TraceInformation($"Bucket {bucketCount}, round {i}: {bufferManager}"); - - Assert.AreEqual(totalBytes, bufferManager.AllocatedBytes); - Assert.AreEqual(bucketCount, bufferManager.BucketCount); - Assert.AreEqual(0, bufferManager.AvailableBytes); - } - } - - bucketCount--; - - int returnedBytes = 0; - foreach (byte[] buffer in buffers) - { - bufferManager.ReturnBuffer(buffer); - - returnedBytes += buffer.Length; - - Trace.TraceInformation($"After returning {returnedBytes} bytes: {bufferManager}"); - Assert.AreEqual(bucketCount, bufferManager.BucketCount); - Assert.AreEqual(returnedBytes, bufferManager.AvailableBytes); - } - } - - /// - /// Verifies that large buffers do not get allocated to the pool. - /// - [TestMethod] - public void LargeBuffer() - { - var bufferManager = new SimpleBufferManager(); - - int bufferSize = SimpleBufferManager.MaxBufferSize + 1; - byte[] buffer = bufferManager.TakeBuffer(bufferSize); - Trace.TraceInformation(bufferManager.ToString()); - Assert.AreEqual(bufferSize, buffer.Length); - Assert.AreEqual(0, bufferManager.AllocatedBytes); - Assert.AreEqual(0, bufferManager.AvailableBytes); - Assert.AreEqual(0, bufferManager.BucketCount); - - bufferManager.ReturnBuffer(buffer); - Trace.TraceInformation(bufferManager.ToString()); - Assert.AreEqual(0, bufferManager.AllocatedBytes); - Assert.AreEqual(0, bufferManager.AvailableBytes); - Assert.AreEqual(0, bufferManager.BucketCount); - } - } -} diff --git a/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj b/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj index 5f0e6dfa2..8bcf2d2a4 100644 --- a/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj +++ b/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj @@ -24,9 +24,9 @@ - - - + + + diff --git a/test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj b/test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj index 3800b3898..9d2e73bc1 100644 --- a/test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj +++ b/test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj @@ -15,9 +15,9 @@ - - - + + + diff --git a/test/DurableTask.Emulator.Tests/DurableTask.Emulator.Tests.csproj b/test/DurableTask.Emulator.Tests/DurableTask.Emulator.Tests.csproj index e123cf09e..f73cb8173 100644 --- a/test/DurableTask.Emulator.Tests/DurableTask.Emulator.Tests.csproj +++ b/test/DurableTask.Emulator.Tests/DurableTask.Emulator.Tests.csproj @@ -6,9 +6,9 @@ - - - + + + diff --git a/test/DurableTask.Redis.Tests/DurableTask.Redis.Tests.csproj b/test/DurableTask.Redis.Tests/DurableTask.Redis.Tests.csproj index 59122e64c..37ac60bdf 100644 --- a/test/DurableTask.Redis.Tests/DurableTask.Redis.Tests.csproj +++ b/test/DurableTask.Redis.Tests/DurableTask.Redis.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/test/DurableTask.ServiceBus.Tests/DurableTask.ServiceBus.Tests.csproj b/test/DurableTask.ServiceBus.Tests/DurableTask.ServiceBus.Tests.csproj index 496621005..d85027fe2 100644 --- a/test/DurableTask.ServiceBus.Tests/DurableTask.ServiceBus.Tests.csproj +++ b/test/DurableTask.ServiceBus.Tests/DurableTask.ServiceBus.Tests.csproj @@ -30,9 +30,9 @@ - - - + + + diff --git a/test/DurableTask.SqlServer.Tests/DurableTask.SqlServer.Tests.csproj b/test/DurableTask.SqlServer.Tests/DurableTask.SqlServer.Tests.csproj index b601fc142..4eed6885c 100644 --- a/test/DurableTask.SqlServer.Tests/DurableTask.SqlServer.Tests.csproj +++ b/test/DurableTask.SqlServer.Tests/DurableTask.SqlServer.Tests.csproj @@ -20,9 +20,9 @@ - - - + + + From eccc9f62aa2c9ecddacc130cbe76603f8ffdd7fb Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Sat, 26 Aug 2023 18:18:35 -0700 Subject: [PATCH 13/62] Fix comment and variable names --- src/DurableTask.AzureStorage/MessageManager.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index 6362d2d2f..2eb19ff9b 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -182,14 +182,14 @@ public async Task CompressAndUploadAsBytesAsync(byte[] payloadBuffer, string blo Blob blob = this.blobContainer.GetBlobReference(blobName); using Stream blobStream = await blob.OpenWriteAsync(cancellationToken); - using GZipStream compressedStream = new GZipStream(blobStream, CompressionLevel.Optimal); + using GZipStream compressedBlobStream = new GZipStream(blobStream, CompressionLevel.Optimal); using MemoryStream payloadStream = new MemoryStream(payloadBuffer); try { - // TODO: Change default pipe options if necessary - await payloadStream.CopyToAsync(compressedStream, bufferSize: 81920, cancellationToken: cancellationToken); - await compressedStream.FlushAsync(cancellationToken); + // Note: 81920 bytes or 80 KB is the default value used by CopyToAsync + await payloadStream.CopyToAsync(compressedBlobStream, bufferSize: 81920, cancellationToken: cancellationToken); + await compressedBlobStream.FlushAsync(cancellationToken); await blobStream.FlushAsync(cancellationToken); } catch (RequestFailedException rfe) @@ -221,8 +221,8 @@ public Task DeleteBlobAsync(string blobName, CancellationToken cancellatio private async Task DownloadAndDecompressAsBytesAsync(Blob blob, CancellationToken cancellationToken = default) { using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken); - using GZipStream decompressedStream = new GZipStream(result.Content, CompressionMode.Decompress); - using StreamReader reader = new StreamReader(decompressedStream, Encoding.UTF8); + using GZipStream decompressedBlobStream = new GZipStream(result.Content, CompressionMode.Decompress); + using StreamReader reader = new StreamReader(decompressedBlobStream, Encoding.UTF8); return await reader.ReadToEndAsync(); } From 519df8c2c935ee3927f253741128a7610a6b08a4 Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Wed, 13 Sep 2023 14:57:45 -0700 Subject: [PATCH 14/62] Update Azure SDK libs --- .../DurableTask.AzureStorage.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index d0eda7cad..04a106f89 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -35,10 +35,10 @@ - + - - + + From e9ed6b6a77cffc3d3fcc831eec291d376534d47e Mon Sep 17 00:00:00 2001 From: Naiyuan Tian <110135109+nytian@users.noreply.github.com> Date: Mon, 25 Sep 2023 09:38:34 -0700 Subject: [PATCH 15/62] Update DurableTask.AzureStorage.csproj (#976) --- src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 04a106f89..b99bbab9a 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -24,7 +24,7 @@ 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) - preview.4 + preview.5 $(VersionPrefix).0 $(VersionPrefix).$(FileVersionRevision) From e292280e3d654a5f45ebd39a7a21881784a83269 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Tue, 19 Sep 2023 10:02:06 -0700 Subject: [PATCH 16/62] Add RawInput to workaround double-serialization issue (#966) --- .../Serializing/DataConverter.cs | 13 ++++++ .../Serializing/Internal/RawInput.cs | 42 +++++++++++++++++++ src/DurableTask.Core/TaskHubClient.cs | 6 +-- 3 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 src/DurableTask.Core/Serializing/Internal/RawInput.cs diff --git a/src/DurableTask.Core/Serializing/DataConverter.cs b/src/DurableTask.Core/Serializing/DataConverter.cs index d2667db25..272e39105 100644 --- a/src/DurableTask.Core/Serializing/DataConverter.cs +++ b/src/DurableTask.Core/Serializing/DataConverter.cs @@ -14,6 +14,7 @@ namespace DurableTask.Core.Serializing { using System; + using DurableTask.Core.Serializing.Internal; /// /// Abstract class for serializing and deserializing data @@ -59,5 +60,17 @@ public T Deserialize(string data) return (T)result; } + + internal string SerializeInternal(object value) + { +#pragma warning disable CS0618 // Type or member is obsolete. Intentional internal usage. + if (value is RawInput raw) + { + return raw.Value; + } +#pragma warning restore CS0618 // Type or member is obsolete + + return this.Serialize(value); + } } } \ No newline at end of file diff --git a/src/DurableTask.Core/Serializing/Internal/RawInput.cs b/src/DurableTask.Core/Serializing/Internal/RawInput.cs new file mode 100644 index 000000000..082dd3501 --- /dev/null +++ b/src/DurableTask.Core/Serializing/Internal/RawInput.cs @@ -0,0 +1,42 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +#nullable enable +using System; + +namespace DurableTask.Core.Serializing.Internal +{ + /// + /// This is an internal API that supports the DurableTask infrastructure and not subject to the same compatibility + /// standards as public APIs. It may be changed or removed without notice in any release. You should only use it + /// directly in your code with extreme caution and knowing that doing so can result in application failures when + /// updating to a new DurableTask release. + /// + [Obsolete("Not for public consumption.")] + public sealed class RawInput + { + /// + /// Initializes a new instance of the class. + /// + /// The raw input value to use. + public RawInput(string? value) + { + this.Value = value; + } + + /// + /// Gets the raw input value. + /// + public string? Value { get; } + } +} diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index c0d215e49..af03ac1a5 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -591,7 +591,7 @@ public Task CreateOrchestrationInstanceAsync(string name, ExecutionId = Guid.NewGuid().ToString("N"), }; - string serializedOrchestrationData = this.defaultConverter.Serialize(orchestrationInput); + string serializedOrchestrationData = this.defaultConverter.SerializeInternal(orchestrationInput); var startedEvent = new ExecutionStartedEvent(-1, serializedOrchestrationData) { Tags = orchestrationTags, @@ -626,7 +626,7 @@ public Task CreateOrchestrationInstanceAsync(string name, if (eventData != null) { - string serializedEventData = this.defaultConverter.Serialize(eventData); + string serializedEventData = this.defaultConverter.SerializeInternal(eventData); var eventRaisedEvent = new EventRaisedEvent(-1, serializedEventData) { Name = eventName }; this.logHelper.RaisingEvent(orchestrationInstance, eventRaisedEvent); @@ -694,7 +694,7 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s throw new ArgumentException(nameof(orchestrationInstance)); } - string serializedInput = this.defaultConverter.Serialize(eventData); + string serializedInput = this.defaultConverter.SerializeInternal(eventData); // Distributed Tracing EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName }; From d0aed6a86f7bb23fafbb5bdfad0ef6ddf6453b12 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Tue, 19 Sep 2023 16:04:26 -0700 Subject: [PATCH 17/62] Use SerializeInternal in TaskOrchestrationContext (#968) --- src/DurableTask.Core/TaskOrchestrationContext.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index a9831ff47..d56b18de6 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -97,7 +97,7 @@ internal void ClearPendingActions() params object[] parameters) { int id = this.idCounter++; - string serializedInput = this.MessageDataConverter.Serialize(parameters); + string serializedInput = this.MessageDataConverter.SerializeInternal(parameters); var scheduleTaskTaskAction = new ScheduleTaskOrchestratorAction { Id = id, @@ -152,7 +152,7 @@ internal void ClearPendingActions() IDictionary tags) { int id = this.idCounter++; - string serializedInput = this.MessageDataConverter.Serialize(input); + string serializedInput = this.MessageDataConverter.SerializeInternal(input); string actualInstanceId = instanceId; if (string.IsNullOrWhiteSpace(actualInstanceId)) @@ -196,7 +196,7 @@ public override void SendEvent(OrchestrationInstance orchestrationInstance, stri } int id = this.idCounter++; - string serializedEventData = this.MessageDataConverter.Serialize(eventData); + string serializedEventData = this.MessageDataConverter.SerializeInternal(eventData); var action = new SendEventOrchestratorAction { @@ -221,7 +221,7 @@ public override void ContinueAsNew(string newVersion, object input) void ContinueAsNewCore(string newVersion, object input) { - string serializedInput = this.MessageDataConverter.Serialize(input); + string serializedInput = this.MessageDataConverter.SerializeInternal(input); this.continueAsNew = new OrchestrationCompleteOrchestratorAction { From ac16db0041fb7c91ada5013d280b60005cd3fc74 Mon Sep 17 00:00:00 2001 From: Varshitha Bachu Date: Wed, 27 Sep 2023 12:23:43 -0700 Subject: [PATCH 18/62] Update DistributedTraceContext (#969) This PR adds a setter for TraceParent to allow it to get serialized. Without this addition, we weren't able to run Netherite apps with the distributed tracing changes because we were seeing this exception: InvalidDataContractException: No set method for property 'TraceParent' in type 'DurableTask.Core.Tracing.DistributedTraceContext'. It also updates the TraceState setter to include this condition - traceState?.Length <= 513 ? traceState : null instead of adding it in the constructor. --- .../Tracing/DistributedTraceContext.cs | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs index 44cbf5ea3..b69b1f5f6 100644 --- a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs +++ b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs @@ -23,6 +23,8 @@ namespace DurableTask.Core.Tracing [DataContract] public class DistributedTraceContext { + private string? traceState; + /// /// Initializes a new instance of the class. /// @@ -31,26 +33,35 @@ public class DistributedTraceContext public DistributedTraceContext(string traceParent, string? traceState = null) { this.TraceParent = traceParent; - - // The W3C spec allows vendors to truncate the trace state if it exceeds 513 characters, - // but it has very specific requirements on HOW trace state can be modified, including - // removing whole values, starting with the largest values, and preserving ordering. - // Rather than implementing these complex requirements, we take the lazy path of just - // truncating the whole thing. - this.TraceState = traceState?.Length <= 513 ? traceState : null; + this.traceState = traceState; } /// /// The W3C traceparent data: https://www.w3.org/TR/trace-context/#traceparent-header /// [DataMember] - public string TraceParent { get; } + public string TraceParent { get; set; } /// /// The optional W3C tracestate parameter: https://www.w3.org/TR/trace-context/#tracestate-header /// [DataMember] - public string? TraceState { get; set; } + public string? TraceState + { + get + { + return this.traceState; + } + set + { + // The W3C spec allows vendors to truncate the trace state if it exceeds 513 characters, + // but it has very specific requirements on HOW trace state can be modified, including + // removing whole values, starting with the largest values, and preserving ordering. + // Rather than implementing these complex requirements, we take the lazy path of just + // truncating the whole thing. + this.traceState = value?.Length <= 513 ? value : null; + } + } /// /// The Activity's Id value that is used to restore an Activity during replays. From 6c8a581a506823098254c2cc43b067092070c9fc Mon Sep 17 00:00:00 2001 From: Varshitha Bachu Date: Wed, 27 Sep 2023 12:57:29 -0700 Subject: [PATCH 19/62] Update ActivitySource in OpenTelemetry sample (#977) --- samples/DistributedTraceSample/OpenTelemetry/Program.cs | 2 +- samples/DistributedTraceSample/OpenTelemetry/README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/DistributedTraceSample/OpenTelemetry/Program.cs b/samples/DistributedTraceSample/OpenTelemetry/Program.cs index 5afcd7500..c200845ee 100644 --- a/samples/DistributedTraceSample/OpenTelemetry/Program.cs +++ b/samples/DistributedTraceSample/OpenTelemetry/Program.cs @@ -29,7 +29,7 @@ static async Task Main(string[] args) { using var tracerProvider = Sdk.CreateTracerProviderBuilder() .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("MySample")) - .AddSource("DurableTask") + .AddSource("DurableTask.Core") .AddConsoleExporter() .AddZipkinExporter() .AddAzureMonitorTraceExporter(options => diff --git a/samples/DistributedTraceSample/OpenTelemetry/README.md b/samples/DistributedTraceSample/OpenTelemetry/README.md index fe6af2469..e9b61a2bf 100644 --- a/samples/DistributedTraceSample/OpenTelemetry/README.md +++ b/samples/DistributedTraceSample/OpenTelemetry/README.md @@ -23,12 +23,12 @@ The following package references are added to OpenTelemetrySample.csproj so that ## Tracer Provider -The following startup code is added to create a tracer provider. This code is necessary to add at startup to ensure that the traces are collected and emitted to the correct telemetry exporters. It specifies the service name for the app, which source that the traces should be collected from, and the telemetry exporters where the traces get emitted. "DurableTask" is the service name that will emit the Durable Task related traces. +The following startup code is added to create a tracer provider. This code is necessary to add at startup to ensure that the traces are collected and emitted to the correct telemetry exporters. It specifies the service name for the app, which source that the traces should be collected from, and the telemetry exporters where the traces get emitted. "DurableTask.Core" is the service name that will emit the Durable Task related traces. ``` using var tracerProvider = Sdk.CreateTracerProviderBuilder() .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("MySample")) - .AddSource("DurableTask") + .AddSource("DurableTask.Core") .AddConsoleExporter() .AddZipkinExporter() .AddAzureMonitorTraceExporter(options => From 343c5676038df368a0a87c29a3081b7bf5472950 Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 9 Oct 2023 19:15:52 -0700 Subject: [PATCH 20/62] Increase DTFx.Core and DTFx.AS versions to 2.15.0 and 1.15.1 respectively (#987) --- src/DurableTask.Core/DurableTask.Core.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/DurableTask.Core.csproj b/src/DurableTask.Core/DurableTask.Core.csproj index 6bf2297ea..beb3a926c 100644 --- a/src/DurableTask.Core/DurableTask.Core.csproj +++ b/src/DurableTask.Core/DurableTask.Core.csproj @@ -17,7 +17,7 @@ 2 - 14 + 15 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) From 43af2a738292fbf77bdc8ab7d1a20f9eac14903d Mon Sep 17 00:00:00 2001 From: Varshitha Bachu Date: Wed, 11 Oct 2023 12:04:23 -0700 Subject: [PATCH 21/62] Update DT.Core and DT.ApplicationInsights versions (#991) DT.Core 2.15.0 --> 2.15.1 DT.ApplicationInsights 0.1.1 --> 0.1.2 --- .../DurableTask.ApplicationInsights.csproj | 2 +- src/DurableTask.Core/DurableTask.Core.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj b/src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj index a8b62fb87..ab23d14f3 100644 --- a/src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj +++ b/src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj @@ -12,7 +12,7 @@ 0 1 - 1 + 2 $(MajorVersion).$(MinorVersion).$(PatchVersion) $(VersionPrefix).0 diff --git a/src/DurableTask.Core/DurableTask.Core.csproj b/src/DurableTask.Core/DurableTask.Core.csproj index beb3a926c..802e5e13f 100644 --- a/src/DurableTask.Core/DurableTask.Core.csproj +++ b/src/DurableTask.Core/DurableTask.Core.csproj @@ -18,7 +18,7 @@ 2 15 - 0 + 1 $(MajorVersion).$(MinorVersion).$(PatchVersion) $(VersionPrefix).0 From 252d0ac37bcd5578fb31ec8e270c288ade17d229 Mon Sep 17 00:00:00 2001 From: Naiyuan Tian <110135109+nytian@users.noreply.github.com> Date: Mon, 16 Oct 2023 10:30:47 -0700 Subject: [PATCH 22/62] Replace STJ package with Newtonsoft (#995) * remove STJ with Newtonsoft * increase DTFx.AS version --- src/DurableTask.AzureStorage/Tracking/TagsSerializer.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DurableTask.AzureStorage/Tracking/TagsSerializer.cs b/src/DurableTask.AzureStorage/Tracking/TagsSerializer.cs index 5fa02a47d..32093266e 100644 --- a/src/DurableTask.AzureStorage/Tracking/TagsSerializer.cs +++ b/src/DurableTask.AzureStorage/Tracking/TagsSerializer.cs @@ -12,16 +12,16 @@ // ---------------------------------------------------------------------------------- using System.Collections.Generic; -using System.Text.Json; +using Newtonsoft.Json; namespace DurableTask.AzureStorage.Tracking { internal static class TagsSerializer { public static string Serialize(IDictionary tags) - => JsonSerializer.Serialize(tags); + => JsonConvert.SerializeObject(tags); public static IDictionary Deserialize(string tags) - => JsonSerializer.Deserialize>(tags); + => JsonConvert.DeserializeObject>(tags); } } From edf4f199c76158a05e35cd04178ce14df90b1fbf Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Sat, 14 Oct 2023 10:18:43 -0700 Subject: [PATCH 23/62] Fix tag serialization --- src/DurableTask.AzureStorage/OrchestrationInstanceStatus.cs | 3 +-- .../Tracking/AzureTableTrackingStore.cs | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/DurableTask.AzureStorage/OrchestrationInstanceStatus.cs b/src/DurableTask.AzureStorage/OrchestrationInstanceStatus.cs index 2cf8d8e96..29dea2c4d 100644 --- a/src/DurableTask.AzureStorage/OrchestrationInstanceStatus.cs +++ b/src/DurableTask.AzureStorage/OrchestrationInstanceStatus.cs @@ -14,7 +14,6 @@ namespace DurableTask.AzureStorage { using System; - using System.Collections.Generic; using Azure; using Azure.Data.Tables; @@ -35,7 +34,7 @@ class OrchestrationInstanceStatus : ITableEntity public string RuntimeStatus { get; set; } public DateTime? ScheduledStartTime { get; set; } public int Generation { get; set; } - public IDictionary Tags { get; set; } + public string Tags { get; set; } public string PartitionKey { get; set; } public string RowKey { get; set; } public DateTimeOffset? Timestamp { get; set; } diff --git a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs index 49c2ed4ae..f602cd4a8 100644 --- a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs @@ -462,7 +462,9 @@ async Task ConvertFromAsync(OrchestrationInstanceStatus orch orchestrationState.Output = orchestrationInstanceStatus.Output; orchestrationState.ScheduledStartTime = orchestrationInstanceStatus.ScheduledStartTime; orchestrationState.Generation = orchestrationInstanceStatus.Generation; - orchestrationState.Tags = orchestrationInstanceStatus.Tags; + orchestrationState.Tags = !string.IsNullOrEmpty(orchestrationInstanceStatus.Tags) + ? TagsSerializer.Deserialize(orchestrationInstanceStatus.Tags) + : null; if (this.settings.FetchLargeMessageDataEnabled) { @@ -716,7 +718,7 @@ public override async Task PurgeInstanceHistoryAsync(string ["ScheduledStartTime"] = executionStartedEvent.ScheduledStartTime, ["ExecutionId"] = executionStartedEvent.OrchestrationInstance.ExecutionId, ["Generation"] = executionStartedEvent.Generation, - ["Tags"] = executionStartedEvent.Tags, + ["Tags"] = TagsSerializer.Serialize(executionStartedEvent.Tags), }; // It is possible that the queue message was small enough to be written directly to a queue message, From bf1e16866e84337979b1c0f3e86d39e055d3e1f5 Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Fri, 17 Nov 2023 15:44:04 -0800 Subject: [PATCH 24/62] Fix Test Execution for Azurite (#1004) * Add new pipeline * Use VSTest * Remove SDK installation * Use Debug to prevent signing * Add config to path * Move Test to test * More movement * Update VSTest task * Install SDK * Fix test attributes * Use latest SDK for build * Change APIs * Nit * Use new API as much as possible * Retrieve instead of query * Remove pipeline --- DurableTask.sln | 3 +- samples/Correlation.Samples/appsettings.json | 2 +- samples/DurableTask.Samples/App.config | 2 +- .../AzureStorageScaleTests.cs | 2 +- .../AzureStorageScenarioTests.cs | 4 +- .../KeySanitationTests.cs | 2 +- .../MessageManagerTests.cs | 0 .../Net/UriPathTests.cs | 0 .../Obsolete/LegacyTableEntityConverter.cs | 0 .../ScheduleTaskTests.cs | 0 .../StressTests.cs | 0 .../TableEntityConverterTests.cs | 61 ++++++++++--------- .../TestHelpers.cs | 2 +- .../TestInstance.cs | 0 .../TestTablePartitionManager.cs | 0 .../ExceptionHandlingIntegrationTests.cs | 0 .../RetryInterceptorTests.cs | 0 test/DurableTask.Core.Tests/app.config | 2 +- .../AzureTableClientTest.cs | 2 +- .../OrchestrationHubTableClientTests.cs | 2 +- .../TestObjectCreator.cs | 0 test/DurableTask.ServiceBus.Tests/app.config | 2 +- .../testhost.dll.config | 2 +- .../DurableTask.Stress.Tests.dll.config | 2 +- test/DurableTask.Stress.Tests/app.config | 2 +- ...impleOrchestrationWithSubOrchestration2.cs | 0 .../SimpleRecurringSubOrchestration.cs | 0 27 files changed, 48 insertions(+), 44 deletions(-) rename {Test => test}/DurableTask.AzureStorage.Tests/MessageManagerTests.cs (100%) rename {Test => test}/DurableTask.AzureStorage.Tests/Net/UriPathTests.cs (100%) rename {Test => test}/DurableTask.AzureStorage.Tests/Obsolete/LegacyTableEntityConverter.cs (100%) rename {Test => test}/DurableTask.AzureStorage.Tests/ScheduleTaskTests.cs (100%) rename {Test => test}/DurableTask.AzureStorage.Tests/StressTests.cs (100%) rename {Test => test}/DurableTask.AzureStorage.Tests/TableEntityConverterTests.cs (90%) rename {Test => test}/DurableTask.AzureStorage.Tests/TestInstance.cs (100%) rename {Test => test}/DurableTask.AzureStorage.Tests/TestTablePartitionManager.cs (100%) rename {Test => test}/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs (100%) rename {Test => test}/DurableTask.Core.Tests/RetryInterceptorTests.cs (100%) rename {Test => test}/DurableTask.ServiceBus.Tests/TestObjectCreator.cs (100%) rename {Test => test}/TestFabricApplication/TestApplication.Common/Orchestrations/SimpleOrchestrationWithSubOrchestration2.cs (100%) rename {Test => test}/TestFabricApplication/TestApplication.Common/Orchestrations/SimpleRecurringSubOrchestration.cs (100%) diff --git a/DurableTask.sln b/DurableTask.sln index 4d57ec5e2..f00ae5252 100644 --- a/DurableTask.sln +++ b/DurableTask.sln @@ -67,6 +67,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Correlation.Samples", "samp EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{D02EF5EF-3D7E-4223-B256-439BAF0C8853}" ProjectSection(SolutionItems) = preProject + azure-pipelines-build.yml = azure-pipelines-build.yml azure-pipelines-release.yml = azure-pipelines-release.yml EndProjectSection EndProject @@ -320,7 +321,7 @@ Global {D818ED4C-29B9-431F-8D09-EE8C82510E25} = {240FA679-D5A7-41CA-BA22-70B45A966088} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {2D63A120-9394-48D9-8CA9-1184364FB854} EnterpriseLibraryConfigurationToolBinariesPath = packages\TransientFaultHandling.Core.5.1.1209.1\lib\NET4 + SolutionGuid = {2D63A120-9394-48D9-8CA9-1184364FB854} EndGlobalSection EndGlobal diff --git a/samples/Correlation.Samples/appsettings.json b/samples/Correlation.Samples/appsettings.json index 50c3c2427..cf9f64f32 100644 --- a/samples/Correlation.Samples/appsettings.json +++ b/samples/Correlation.Samples/appsettings.json @@ -1,4 +1,4 @@ { - "StorageConnectionString": "UseDevelopmentStorage=true;DevelopmentStorageProxyUri=http://127.0.0.1:10002/", + "StorageConnectionString": "UseDevelopmentStorage=true", "taskHubName":"SamplesHub" } \ No newline at end of file diff --git a/samples/DurableTask.Samples/App.config b/samples/DurableTask.Samples/App.config index ffe1c8b05..67da5a604 100644 --- a/samples/DurableTask.Samples/App.config +++ b/samples/DurableTask.Samples/App.config @@ -6,7 +6,7 @@ - + diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScaleTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScaleTests.cs index e902f2b7d..18417b96d 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScaleTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScaleTests.cs @@ -201,7 +201,7 @@ private async Task EnsureLeasesMatchControlQueue(string directoryReference, Blob /// REQUIREMENT: Workers can be added or removed at any time and control-queue partitions are load-balanced automatically. /// REQUIREMENT: No two workers will ever process the same control queue. /// - [TestMethod] + [DataTestMethod] [DataRow(PartitionManagerType.V1Legacy, 30)] [DataRow(PartitionManagerType.V2Safe, 180)] public async Task MultiWorkerLeaseMovement(PartitionManagerType partitionManagerType, int timeoutInSeconds) diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 84876f0b2..8e8f7ac69 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -900,7 +900,7 @@ public async Task TerminateOrchestration(bool enableExtendedSessions) /// /// End-to-end test which validates the Suspend-Resume functionality. /// - [TestMethod] + [DataTestMethod] [DataRow(true)] [DataRow(false)] public async Task SuspendResumeOrchestration(bool enableExtendedSessions) @@ -939,7 +939,7 @@ public async Task SuspendResumeOrchestration(bool enableExtendedSessions) /// /// Test that a suspended orchestration can be terminated. /// - [TestMethod] + [DataTestMethod] [DataRow(true)] [DataRow(false)] public async Task TerminateSuspendedOrchestration(bool enableExtendedSessions) diff --git a/test/DurableTask.AzureStorage.Tests/KeySanitationTests.cs b/test/DurableTask.AzureStorage.Tests/KeySanitationTests.cs index d631241ac..326ffc1e3 100644 --- a/test/DurableTask.AzureStorage.Tests/KeySanitationTests.cs +++ b/test/DurableTask.AzureStorage.Tests/KeySanitationTests.cs @@ -24,7 +24,7 @@ namespace DurableTask.AzureStorage.Tests [TestClass] public class KeySanitationTests { - [TestMethod] + [DataTestMethod] [DataRow("\r")] [DataRow("")] [DataRow("hello")] diff --git a/Test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs similarity index 100% rename from Test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs rename to test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs diff --git a/Test/DurableTask.AzureStorage.Tests/Net/UriPathTests.cs b/test/DurableTask.AzureStorage.Tests/Net/UriPathTests.cs similarity index 100% rename from Test/DurableTask.AzureStorage.Tests/Net/UriPathTests.cs rename to test/DurableTask.AzureStorage.Tests/Net/UriPathTests.cs diff --git a/Test/DurableTask.AzureStorage.Tests/Obsolete/LegacyTableEntityConverter.cs b/test/DurableTask.AzureStorage.Tests/Obsolete/LegacyTableEntityConverter.cs similarity index 100% rename from Test/DurableTask.AzureStorage.Tests/Obsolete/LegacyTableEntityConverter.cs rename to test/DurableTask.AzureStorage.Tests/Obsolete/LegacyTableEntityConverter.cs diff --git a/Test/DurableTask.AzureStorage.Tests/ScheduleTaskTests.cs b/test/DurableTask.AzureStorage.Tests/ScheduleTaskTests.cs similarity index 100% rename from Test/DurableTask.AzureStorage.Tests/ScheduleTaskTests.cs rename to test/DurableTask.AzureStorage.Tests/ScheduleTaskTests.cs diff --git a/Test/DurableTask.AzureStorage.Tests/StressTests.cs b/test/DurableTask.AzureStorage.Tests/StressTests.cs similarity index 100% rename from Test/DurableTask.AzureStorage.Tests/StressTests.cs rename to test/DurableTask.AzureStorage.Tests/StressTests.cs diff --git a/Test/DurableTask.AzureStorage.Tests/TableEntityConverterTests.cs b/test/DurableTask.AzureStorage.Tests/TableEntityConverterTests.cs similarity index 90% rename from Test/DurableTask.AzureStorage.Tests/TableEntityConverterTests.cs rename to test/DurableTask.AzureStorage.Tests/TableEntityConverterTests.cs index 358ea8569..f0c4a5c1f 100644 --- a/Test/DurableTask.AzureStorage.Tests/TableEntityConverterTests.cs +++ b/test/DurableTask.AzureStorage.Tests/TableEntityConverterTests.cs @@ -106,21 +106,21 @@ public void DeserializeNull() Assert.IsNull(actual.NullableEnumProperty); Assert.IsNull(actual.StringProperty); Assert.IsNull(actual.BinaryProperty); - Assert.AreEqual(default(bool), actual.BoolProperty); + Assert.AreEqual(default, actual.BoolProperty); Assert.IsNull(actual.NullableBoolProperty); - Assert.AreEqual(default(DateTime), actual.Timestamp); + Assert.AreEqual(default, actual.Timestamp); Assert.IsNull(actual.NullableDateTimeField); - Assert.AreEqual(default(DateTimeOffset), actual.DateTimeOffsetProperty); + Assert.AreEqual(default, actual.DateTimeOffsetProperty); Assert.IsNull(actual.NullableDateTimeOffsetProperty); - Assert.AreEqual(default(double), actual.DoubleField); + Assert.AreEqual(default, actual.DoubleField); Assert.IsNull(actual.NullableDoubleProperty); - Assert.AreEqual(default(Guid), actual.GuidProperty); + Assert.AreEqual(default, actual.GuidProperty); Assert.IsNull(actual.NullableGuidField); - Assert.AreEqual(default(int), actual.IntField); + Assert.AreEqual(default, actual.IntField); Assert.IsNull(actual.NullableIntField); - Assert.AreEqual(default(long), actual.LongField); + Assert.AreEqual(default, actual.LongField); Assert.IsNull(actual.NullableLongProperty); - Assert.AreEqual(default(short), actual.UnsupportedProperty); + Assert.AreEqual(default, actual.UnsupportedProperty); Assert.IsNull(actual.ObjectProperty); } @@ -227,24 +227,26 @@ public async Task BackwardsCompatible() .CreateCloudTableClient() .GetTableReference(nameof(BackwardsCompatible)); + var tableClient = new TableServiceClient(TestHelpers.GetTestStorageAccountConnectionString()) + .GetTableClient(nameof(BackwardsCompatible)); + try { // Initialize table and add the entity - await legacyTableClient.DeleteIfExistsAsync(); - await legacyTableClient.CreateAsync(); + await tableClient.DeleteAsync(); + await tableClient.CreateAsync(); await legacyTableClient.ExecuteAsync(Microsoft.WindowsAzure.Storage.Table.TableOperation.Insert(entity)); // Read the old entity using the new logic - var tableClient = new TableServiceClient(TestHelpers.GetTestStorageAccountConnectionString()).GetTableClient(nameof(BackwardsCompatible)); - var result = await tableClient.QueryAsync(filter: $"{nameof(ITableEntity.RowKey)} eq '1'").SingleAsync(); + var result = await tableClient.GetEntityAsync(entity.PartitionKey, entity.RowKey); // Compare expected.Skipped = null; - Assert.AreEqual(expected, (Example)TableEntityConverter.Deserialize(result, typeof(Example))); + Assert.AreEqual(expected, (Example)TableEntityConverter.Deserialize(result.Value, typeof(Example))); } finally { - await legacyTableClient.DeleteIfExistsAsync(); + await tableClient.DeleteAsync(); } } @@ -286,32 +288,33 @@ public async Task ForwardsCompatible() entity.PartitionKey = "12345"; entity.RowKey = "1"; - var tableClient = new TableServiceClient(TestHelpers.GetTestStorageAccountConnectionString()).GetTableClient(nameof(ForwardsCompatible)); + var legacyTableClient = CloudStorageAccount + .Parse(TestHelpers.GetTestStorageAccountConnectionString()) + .CreateCloudTableClient() + .GetTableReference(nameof(ForwardsCompatible)); + + var tableClient = new TableServiceClient(TestHelpers.GetTestStorageAccountConnectionString()) + .GetTableClient(nameof(ForwardsCompatible)); try { - // Initialize table and add the entity + // Initialize table and add the entity using the latest API await tableClient.DeleteAsync(); await tableClient.CreateAsync(); await tableClient.AddEntityAsync(entity); - // Read the new entity using the old logic - var legacyTableClient = CloudStorageAccount - .Parse(TestHelpers.GetTestStorageAccountConnectionString()) - .CreateCloudTableClient() - .GetTableReference(nameof(ForwardsCompatible)); + // Read the entity using the old API + Microsoft.WindowsAzure.Storage.Table.TableResult response = await legacyTableClient.ExecuteAsync( + Microsoft.WindowsAzure.Storage.Table.TableOperation.Retrieve( + entity.PartitionKey, + entity.RowKey)); - var segment = await legacyTableClient.ExecuteQuerySegmentedAsync( - new Microsoft.WindowsAzure.Storage.Table.TableQuery().Where( - Microsoft.WindowsAzure.Storage.Table.TableQuery.GenerateFilterCondition( - nameof(ITableEntity.RowKey), - Microsoft.WindowsAzure.Storage.Table.QueryComparisons.Equal, - "1")), - null); + var actual = response.Result as Microsoft.WindowsAzure.Storage.Table.DynamicTableEntity; + Assert.IsNotNull(actual); // Compare expected.Skipped = null; - Assert.AreEqual(expected, (Example)new LegacyTableEntityConverter().ConvertFromTableEntity(segment.Single(), x => typeof(Example))); + Assert.AreEqual(expected, (Example)new LegacyTableEntityConverter().ConvertFromTableEntity(actual, x => typeof(Example))); } finally { diff --git a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs index 32cc7f714..3608d0120 100644 --- a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs +++ b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs @@ -55,7 +55,7 @@ public static string GetTestStorageAccountConnectionString() string? storageConnectionString = GetTestSetting("StorageConnectionString"); if (string.IsNullOrEmpty(storageConnectionString)) { - storageConnectionString = "UseDevelopmentStorage=true;DevelopmentStorageProxyUri=http://127.0.0.1:10002/"; + storageConnectionString = "UseDevelopmentStorage=true"; } return storageConnectionString!; diff --git a/Test/DurableTask.AzureStorage.Tests/TestInstance.cs b/test/DurableTask.AzureStorage.Tests/TestInstance.cs similarity index 100% rename from Test/DurableTask.AzureStorage.Tests/TestInstance.cs rename to test/DurableTask.AzureStorage.Tests/TestInstance.cs diff --git a/Test/DurableTask.AzureStorage.Tests/TestTablePartitionManager.cs b/test/DurableTask.AzureStorage.Tests/TestTablePartitionManager.cs similarity index 100% rename from Test/DurableTask.AzureStorage.Tests/TestTablePartitionManager.cs rename to test/DurableTask.AzureStorage.Tests/TestTablePartitionManager.cs diff --git a/Test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs b/test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs similarity index 100% rename from Test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs rename to test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs diff --git a/Test/DurableTask.Core.Tests/RetryInterceptorTests.cs b/test/DurableTask.Core.Tests/RetryInterceptorTests.cs similarity index 100% rename from Test/DurableTask.Core.Tests/RetryInterceptorTests.cs rename to test/DurableTask.Core.Tests/RetryInterceptorTests.cs diff --git a/test/DurableTask.Core.Tests/app.config b/test/DurableTask.Core.Tests/app.config index c9cb6243c..d1d912b64 100644 --- a/test/DurableTask.Core.Tests/app.config +++ b/test/DurableTask.Core.Tests/app.config @@ -1,7 +1,7 @@  - + diff --git a/test/DurableTask.ServiceBus.Tests/AzureTableClientTest.cs b/test/DurableTask.ServiceBus.Tests/AzureTableClientTest.cs index 418755cd4..5eb724f94 100644 --- a/test/DurableTask.ServiceBus.Tests/AzureTableClientTest.cs +++ b/test/DurableTask.ServiceBus.Tests/AzureTableClientTest.cs @@ -21,7 +21,7 @@ namespace DurableTask.ServiceBus.Tests [TestClass] public class AzureTableClientTest { - const string ConnectionString = "UseDevelopmentStorage=true;DevelopmentStorageProxyUri=http://myProxyUri"; + const string ConnectionString = "UseDevelopmentStorage=true"; [TestMethod] public void CreateQueryWithoutFilter() diff --git a/test/DurableTask.ServiceBus.Tests/OrchestrationHubTableClientTests.cs b/test/DurableTask.ServiceBus.Tests/OrchestrationHubTableClientTests.cs index 88f184e5e..7f8407ccb 100644 --- a/test/DurableTask.ServiceBus.Tests/OrchestrationHubTableClientTests.cs +++ b/test/DurableTask.ServiceBus.Tests/OrchestrationHubTableClientTests.cs @@ -36,7 +36,7 @@ public void TestInitialize() { var r = new Random(); this.tableClient = new AzureTableClient("test00" + r.Next(0, 10000), - "UseDevelopmentStorage=true;DevelopmentStorageProxyUri=http://127.0.0.1:10002/"); + "UseDevelopmentStorage=true"); this.tableClient.CreateTableIfNotExistsAsync().Wait(); this.client = TestHelpers.CreateTaskHubClient(); diff --git a/Test/DurableTask.ServiceBus.Tests/TestObjectCreator.cs b/test/DurableTask.ServiceBus.Tests/TestObjectCreator.cs similarity index 100% rename from Test/DurableTask.ServiceBus.Tests/TestObjectCreator.cs rename to test/DurableTask.ServiceBus.Tests/TestObjectCreator.cs diff --git a/test/DurableTask.ServiceBus.Tests/app.config b/test/DurableTask.ServiceBus.Tests/app.config index f98565623..fb698f72a 100644 --- a/test/DurableTask.ServiceBus.Tests/app.config +++ b/test/DurableTask.ServiceBus.Tests/app.config @@ -1,7 +1,7 @@  - + diff --git a/test/DurableTask.ServiceBus.Tests/testhost.dll.config b/test/DurableTask.ServiceBus.Tests/testhost.dll.config index f61f18af3..b3b5e73b5 100644 --- a/test/DurableTask.ServiceBus.Tests/testhost.dll.config +++ b/test/DurableTask.ServiceBus.Tests/testhost.dll.config @@ -1,7 +1,7 @@  - + diff --git a/test/DurableTask.Stress.Tests/DurableTask.Stress.Tests.dll.config b/test/DurableTask.Stress.Tests/DurableTask.Stress.Tests.dll.config index f3b56052a..9bce5f1b3 100644 --- a/test/DurableTask.Stress.Tests/DurableTask.Stress.Tests.dll.config +++ b/test/DurableTask.Stress.Tests/DurableTask.Stress.Tests.dll.config @@ -12,6 +12,6 @@ - + diff --git a/test/DurableTask.Stress.Tests/app.config b/test/DurableTask.Stress.Tests/app.config index 4b4b89436..3f2787681 100644 --- a/test/DurableTask.Stress.Tests/app.config +++ b/test/DurableTask.Stress.Tests/app.config @@ -12,6 +12,6 @@ - + diff --git a/Test/TestFabricApplication/TestApplication.Common/Orchestrations/SimpleOrchestrationWithSubOrchestration2.cs b/test/TestFabricApplication/TestApplication.Common/Orchestrations/SimpleOrchestrationWithSubOrchestration2.cs similarity index 100% rename from Test/TestFabricApplication/TestApplication.Common/Orchestrations/SimpleOrchestrationWithSubOrchestration2.cs rename to test/TestFabricApplication/TestApplication.Common/Orchestrations/SimpleOrchestrationWithSubOrchestration2.cs diff --git a/Test/TestFabricApplication/TestApplication.Common/Orchestrations/SimpleRecurringSubOrchestration.cs b/test/TestFabricApplication/TestApplication.Common/Orchestrations/SimpleRecurringSubOrchestration.cs similarity index 100% rename from Test/TestFabricApplication/TestApplication.Common/Orchestrations/SimpleRecurringSubOrchestration.cs rename to test/TestFabricApplication/TestApplication.Common/Orchestrations/SimpleRecurringSubOrchestration.cs From bdea4c3c73376e370b9b488b6abdc9a8965e4fdf Mon Sep 17 00:00:00 2001 From: Naiyuan Tian <110135109+nytian@users.noreply.github.com> Date: Tue, 7 May 2024 14:19:32 -0400 Subject: [PATCH 25/62] update with ASTrack2 methods (#1081) --- .../EntityTrackingStoreQueries.cs | 17 ++++++++++++----- ...OrchestrationInstanceStatusQueryCondition.cs | 5 +---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/DurableTask.AzureStorage/EntityTrackingStoreQueries.cs b/src/DurableTask.AzureStorage/EntityTrackingStoreQueries.cs index 3ba243908..1bf9bfadb 100644 --- a/src/DurableTask.AzureStorage/EntityTrackingStoreQueries.cs +++ b/src/DurableTask.AzureStorage/EntityTrackingStoreQueries.cs @@ -21,6 +21,7 @@ namespace DurableTask.AzureStorage using System.Text; using System.Threading; using System.Threading.Tasks; + using Azure; using DurableTask.AzureStorage.Tracking; using DurableTask.Core; using DurableTask.Core.Entities; @@ -56,7 +57,7 @@ class EntityTrackingStoreQueries : EntityBackendQueries CancellationToken cancellation = default(CancellationToken)) { await this.ensureTaskHub(); - OrchestrationState? state = (await this.trackingStore.GetStateAsync(id.ToString(), allExecutions: false, fetchInput: includeState)).FirstOrDefault(); + OrchestrationState? state = await this.trackingStore.GetStateAsync(id.ToString(), allExecutions: false, fetchInput: includeState).FirstOrDefaultAsync(); return await this.GetEntityMetadataAsync(state, includeStateless, includeState); } @@ -87,7 +88,10 @@ public async override Task QueryEntitiesAsync(EntityQuery fil do { - DurableStatusQueryResult result = await this.trackingStore.GetStateAsync(condition, filter.PageSize ?? 100, continuationToken, cancellation); + Page? page = await this.trackingStore.GetStateAsync(condition, cancellation).AsPages(continuationToken, filter.PageSize ?? 100).FirstOrDefaultAsync(); + DurableStatusQueryResult result = page != null + ? new DurableStatusQueryResult { ContinuationToken = page.ContinuationToken, OrchestrationState = page.Values } + : new DurableStatusQueryResult { OrchestrationState = Array.Empty() }; entityResult = await ConvertResultsAsync(result.OrchestrationState); continuationToken = result.ContinuationToken; } @@ -139,7 +143,10 @@ public async override Task CleanEntityStorageAsync(Cle // perform that action. Waits for all actions to finish after each page. do { - DurableStatusQueryResult page = await this.trackingStore.GetStateAsync(condition, 100, continuationToken, cancellation); + Page? states = await this.trackingStore.GetStateAsync(condition, cancellation).AsPages(continuationToken, 100).FirstOrDefaultAsync(); + DurableStatusQueryResult page = states != null + ? new DurableStatusQueryResult { ContinuationToken = states.ContinuationToken, OrchestrationState = states.Values } + : new DurableStatusQueryResult { OrchestrationState = Array.Empty() }; continuationToken = page.ContinuationToken; var tasks = new List(); @@ -174,8 +181,8 @@ async Task DeleteIdleOrchestrationEntity(OrchestrationState state) async Task CheckForOrphanedLockAndFixIt(OrchestrationState state, string lockOwner) { - OrchestrationState? ownerState - = (await this.trackingStore.GetStateAsync(lockOwner, allExecutions: false, fetchInput: false)).FirstOrDefault(); + OrchestrationState? ownerState + = await this.trackingStore.GetStateAsync(lockOwner, allExecutions: false, fetchInput: false).FirstOrDefaultAsync(); bool OrchestrationIsRunning(OrchestrationStatus? status) => status != null && (status == OrchestrationStatus.Running || status == OrchestrationStatus.Suspended); diff --git a/src/DurableTask.AzureStorage/Tracking/OrchestrationInstanceStatusQueryCondition.cs b/src/DurableTask.AzureStorage/Tracking/OrchestrationInstanceStatusQueryCondition.cs index 9ef2c20ca..0f47a3979 100644 --- a/src/DurableTask.AzureStorage/Tracking/OrchestrationInstanceStatusQueryCondition.cs +++ b/src/DurableTask.AzureStorage/Tracking/OrchestrationInstanceStatusQueryCondition.cs @@ -145,10 +145,7 @@ internal ODataCondition ToOData() } else if (this.ExcludeEntities) { - conditions.Add(TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThan, "@"), - TableOperators.Or, - TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, "A"))); + conditions.Add($"{nameof(OrchestrationInstanceStatus.PartitionKey)} lt '@' or {nameof(OrchestrationInstanceStatus.PartitionKey)} ge 'A'"); } if (this.InstanceId != null) From 5d0c40017bc5e0b6f7f538e17b4c3e64ffbc21e0 Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Mon, 20 May 2024 14:30:35 -0700 Subject: [PATCH 26/62] solve conflicts when merge with main --- .../DurableTask.ApplicationInsights.csproj | 4 ---- .../DurableTask.AzureStorage.csproj | 7 ------- 2 files changed, 11 deletions(-) diff --git a/src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj b/src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj index 438891729..0bccf5929 100644 --- a/src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj +++ b/src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj @@ -12,11 +12,7 @@ 0 1 -<<<<<<< HEAD 5 -======= - 2 ->>>>>>> origin/azure-storage-v12 $(MajorVersion).$(MinorVersion).$(PatchVersion) $(VersionPrefix).0 diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 2b50f9e34..4bbd8a79d 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -19,16 +19,9 @@ -<<<<<<< HEAD - 1 - 17 - 2 -======= 2 0 0 - ->>>>>>> origin/azure-storage-v12 $(MajorVersion).$(MinorVersion).$(PatchVersion) preview.5 $(VersionPrefix).0 From 8700b5db1741fdbb9e4fa039eb915949c21a239d Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Mon, 20 May 2024 14:50:34 -0700 Subject: [PATCH 27/62] remove durabletask.redis.tests.csproj --- .../DurableTask.Redis.Tests.csproj | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 test/DurableTask.Redis.Tests/DurableTask.Redis.Tests.csproj diff --git a/test/DurableTask.Redis.Tests/DurableTask.Redis.Tests.csproj b/test/DurableTask.Redis.Tests/DurableTask.Redis.Tests.csproj deleted file mode 100644 index 37ac60bdf..000000000 --- a/test/DurableTask.Redis.Tests/DurableTask.Redis.Tests.csproj +++ /dev/null @@ -1,24 +0,0 @@ - - - - netcoreapp3.1 - false - - - - - - - - - - - - - - - - - - - From e50f3de6e2517821ec64e66a38195f6e0137c5cf Mon Sep 17 00:00:00 2001 From: Naiyuan Tian <110135109+nytian@users.noreply.github.com> Date: Wed, 22 May 2024 18:56:38 -0700 Subject: [PATCH 28/62] Update DurableTask.AzureStorage.csproj (#1098) --- src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 4bbd8a79d..243ebbca2 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -23,7 +23,7 @@ 0 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) - preview.5 + preview.6 $(VersionPrefix).0 $(VersionPrefix).$(FileVersionRevision) From e235944ecb4332cfbf5d2d2ecbda10d6b96dda50 Mon Sep 17 00:00:00 2001 From: Naiyuan Tian <110135109+nytian@users.noreply.github.com> Date: Thu, 23 May 2024 13:28:39 -0700 Subject: [PATCH 29/62] Upgrade Azure Storage SDK to the Latest (#1099) * Update DurableTask.AzureStorage.csproj * Update DurableTask.AzureStorage.csproj --- .../DurableTask.AzureStorage.csproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 243ebbca2..ab969b27e 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -40,10 +40,10 @@ - - - - + + + + From 0aa8886d8ffefdebdf86d39eab23400ac9db61fc Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Tue, 28 May 2024 13:53:07 -0700 Subject: [PATCH 30/62] solve conflicts --- .../DurableTask.AzureStorage.csproj | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index dae9fe60f..ab969b27e 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -19,15 +19,9 @@ -<<<<<<< HEAD - 1 - 17 - 3 -======= 2 0 0 ->>>>>>> azure-storage-v12 $(MajorVersion).$(MinorVersion).$(PatchVersion) preview.6 $(VersionPrefix).0 From 584babe7683e4f1df717372ba44e0646145d965c Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Wed, 29 May 2024 10:49:30 -0700 Subject: [PATCH 31/62] Update Pop Receipt (#1066) * WIP * Refactor * nit --- src/DurableTask.AzureStorage/MessageData.cs | 5 +++++ .../Messaging/ControlQueue.cs | 5 +++-- .../Messaging/TaskHubQueue.cs | 22 ++++++++++++++----- src/DurableTask.AzureStorage/Storage/Queue.cs | 5 +++-- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/DurableTask.AzureStorage/MessageData.cs b/src/DurableTask.AzureStorage/MessageData.cs index bf39fbc72..d89e7c686 100644 --- a/src/DurableTask.AzureStorage/MessageData.cs +++ b/src/DurableTask.AzureStorage/MessageData.cs @@ -106,6 +106,11 @@ public MessageData() internal long TotalMessageSizeBytes { get; set; } internal MessageFormatFlags MessageFormat { get; set; } + + internal void Update(UpdateReceipt receipt) + { + this.OriginalQueueMessage = this.OriginalQueueMessage.Update(receipt); + } } /// diff --git a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs index 75f289b3b..9f1c0d2ba 100644 --- a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs +++ b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs @@ -126,7 +126,8 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) e.ToString()); // Abandon the message so we can try it again later. - await this.AbandonMessageAsync(queueMessage); + // Note: We will fetch the message again from the queue before retrying, so no need to read the receipt + _ = await this.AbandonMessageAsync(queueMessage); return; } @@ -191,7 +192,7 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) } // This overload is intended for cases where we aren't able to deserialize an instance of MessageData. - public Task AbandonMessageAsync(QueueMessage queueMessage) + public Task AbandonMessageAsync(QueueMessage queueMessage) { this.stats.PendingOrchestratorMessages.TryRemove(queueMessage.MessageId, out _); return base.AbandonMessageAsync( diff --git a/src/DurableTask.AzureStorage/Messaging/TaskHubQueue.cs b/src/DurableTask.AzureStorage/Messaging/TaskHubQueue.cs index 3188dcec3..d204268bf 100644 --- a/src/DurableTask.AzureStorage/Messaging/TaskHubQueue.cs +++ b/src/DurableTask.AzureStorage/Messaging/TaskHubQueue.cs @@ -212,22 +212,29 @@ async Task AddMessageAsync(TaskMessage taskMessage, OrchestrationIn return initialVisibilityDelay; } - public virtual Task AbandonMessageAsync(MessageData message, SessionBase? session = null) + public virtual async Task AbandonMessageAsync(MessageData message, SessionBase? session = null) { QueueMessage queueMessage = message.OriginalQueueMessage; TaskMessage taskMessage = message.TaskMessage; OrchestrationInstance instance = taskMessage.OrchestrationInstance; long sequenceNumber = message.SequenceNumber; - return this.AbandonMessageAsync( + UpdateReceipt? receipt = await this.AbandonMessageAsync( queueMessage, taskMessage, instance, session?.TraceActivityId, sequenceNumber); + + // If we've successfully abandoned the message, update the pop receipt + // (even though we'll likely no longer interact with this message) + if (receipt is not null) + { + message.Update(receipt); + } } - protected async Task AbandonMessageAsync( + protected async Task AbandonMessageAsync( QueueMessage queueMessage, TaskMessage? taskMessage, OrchestrationInstance? instance, @@ -276,7 +283,7 @@ public virtual Task AbandonMessageAsync(MessageData message, SessionBase? sessio { // We "abandon" the message by settings its visibility timeout using an exponential backoff algorithm. // This allows it to be reprocessed on this node or another node at a later time, hopefully successfully. - await this.storageQueue.UpdateMessageAsync( + return await this.storageQueue.UpdateMessageAsync( queueMessage, TimeSpan.FromSeconds(numSecondsToWait), traceActivityId); @@ -293,6 +300,8 @@ public virtual Task AbandonMessageAsync(MessageData message, SessionBase? sessio taskEventId, details: $"Caller: {nameof(AbandonMessageAsync)}", queueMessage.PopReceipt); + + return null; } } @@ -316,10 +325,13 @@ public async Task RenewMessageAsync(MessageData message, SessionBase session) try { - await this.storageQueue.UpdateMessageAsync( + UpdateReceipt receipt = await this.storageQueue.UpdateMessageAsync( queueMessage, this.MessageVisibilityTimeout, session?.TraceActivityId); + + // Update the pop receipt + message.Update(receipt); } catch (Exception e) { diff --git a/src/DurableTask.AzureStorage/Storage/Queue.cs b/src/DurableTask.AzureStorage/Storage/Queue.cs index 46b4115d8..eca1f1c28 100644 --- a/src/DurableTask.AzureStorage/Storage/Queue.cs +++ b/src/DurableTask.AzureStorage/Storage/Queue.cs @@ -59,10 +59,10 @@ await this.queueClient this.stats.MessagesSent.Increment(); } - public async Task UpdateMessageAsync(QueueMessage queueMessage, TimeSpan visibilityTimeout, Guid? clientRequestId = null, CancellationToken cancellationToken = default) + public async Task UpdateMessageAsync(QueueMessage queueMessage, TimeSpan visibilityTimeout, Guid? clientRequestId = null, CancellationToken cancellationToken = default) { using IDisposable scope = OperationContext.CreateClientRequestScope(clientRequestId); - await this.queueClient + UpdateReceipt receipt = await this.queueClient .UpdateMessageAsync( queueMessage.MessageId, queueMessage.PopReceipt, @@ -71,6 +71,7 @@ await this.queueClient .DecorateFailure(); this.stats.MessagesUpdated.Increment(); + return receipt; } public async Task DeleteMessageAsync(QueueMessage queueMessage, Guid? clientRequestId = null, CancellationToken cancellationToken = default) From 38c4880b934818bbaa9154de29e24a755698f4fd Mon Sep 17 00:00:00 2001 From: Naiyuan Tian <110135109+nytian@users.noreply.github.com> Date: Thu, 13 Jun 2024 21:34:13 -0400 Subject: [PATCH 32/62] HotFixes: Enable Support for Forward Compatibility between DTFx.AzureStorage 1.x to 2.x (#1116) * initial commit * add exception detail --- src/DurableTask.AzureStorage/MessageManager.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index 81ceff7d3..3052146b8 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -156,7 +156,17 @@ public async Task DeserializeQueueMessageAsync(QueueMessage queueMe { // TODO: Deserialize with Stream? byte[] body = queueMessage.Body.ToArray(); - MessageData envelope = this.DeserializeMessageData(Encoding.UTF8.GetString(body)); + MessageData envelope; + try + { + envelope = this.DeserializeMessageData(Encoding.UTF8.GetString(body)); + } + catch(JsonReaderException) + { + // This catch block is a hotfix and better implementation might be needed in future. + // DTFx.AzureStorage 1.x and 2.x use different encoding methods. Adding this line to enable forward compatibility. + envelope = this.DeserializeMessageData(Encoding.UTF8.GetString(Convert.FromBase64String(Encoding.UTF8.GetString(body)))); + } if (!string.IsNullOrEmpty(envelope.CompressedBlobName)) { From b602d788a0e860bcfc93d6fd8aa2db1b9701e74f Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Tue, 25 Jun 2024 20:19:44 -0700 Subject: [PATCH 33/62] delete queuemessage.cs --- .../Storage/QueueMessage.cs | 60 ------------------- 1 file changed, 60 deletions(-) delete mode 100644 src/DurableTask.AzureStorage/Storage/QueueMessage.cs diff --git a/src/DurableTask.AzureStorage/Storage/QueueMessage.cs b/src/DurableTask.AzureStorage/Storage/QueueMessage.cs deleted file mode 100644 index 667284c08..000000000 --- a/src/DurableTask.AzureStorage/Storage/QueueMessage.cs +++ /dev/null @@ -1,60 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- -#nullable enable -namespace DurableTask.AzureStorage.Storage -{ - using System; - using Microsoft.WindowsAzure.Storage.Queue; - - class QueueMessage - { - public QueueMessage(CloudQueueMessage cloudQueueMessage) - { - this.CloudQueueMessage = cloudQueueMessage; - try - { - this.Message = this.CloudQueueMessage.AsString; - } - catch (FormatException) - { - // This try-catch block ensures forwards compatibility with DTFx.AzureStorage v2.x, which does not guarantee base64 encoding of messages (messages not encoded at all). - // Therefore, if we try to decode those messages as base64, we will have a format exception that will yield a poison message - // RawString is an internal property of CloudQueueMessage, so we need to obtain it via reflection. - System.Reflection.PropertyInfo rawStringProperty = typeof(CloudQueueMessage).GetProperty("RawString", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); - this.Message = (string)rawStringProperty.GetValue(this.CloudQueueMessage); - } - this.Id = this.CloudQueueMessage.Id; - } - - public QueueMessage(string message) - { - this.CloudQueueMessage = new CloudQueueMessage(message); - this.Message = this.CloudQueueMessage.AsString; - this.Id = this.CloudQueueMessage.Id; - } - - public CloudQueueMessage CloudQueueMessage { get; } - - public string Message { get; } - - public string Id { get; } - - public int DequeueCount => this.CloudQueueMessage.DequeueCount; - - public DateTimeOffset? InsertionTime => this.CloudQueueMessage.InsertionTime; - - public DateTimeOffset? NextVisibleTime => this.CloudQueueMessage.NextVisibleTime; - - public string PopReceipt => this.CloudQueueMessage.PopReceipt; - } -} From a1605ae2637cca31b8acdda9976f6685ef212f11 Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Tue, 2 Jul 2024 14:51:22 -0700 Subject: [PATCH 34/62] Fix Azure Storage HNS Support (#1123) --- .../DurableTask.AzureStorage.csproj | 2 +- .../Linq/AsyncEnumerableExtensions.cs | 140 ++++++++++++++++++ .../Storage/AsyncPageableAsyncProjection.cs | 62 -------- .../Storage/AsyncPageableProjection.cs | 59 -------- .../Storage/BlobContainer.cs | 23 ++- .../Storage/TableQueryResponse.cs | 21 +-- .../Tracking/AzureTableTrackingStore.cs | 12 +- 7 files changed, 175 insertions(+), 144 deletions(-) create mode 100644 src/DurableTask.AzureStorage/Linq/AsyncEnumerableExtensions.cs delete mode 100644 src/DurableTask.AzureStorage/Storage/AsyncPageableAsyncProjection.cs delete mode 100644 src/DurableTask.AzureStorage/Storage/AsyncPageableProjection.cs diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index e6bbea7f8..81aa6bae6 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -40,7 +40,7 @@ - + diff --git a/src/DurableTask.AzureStorage/Linq/AsyncEnumerableExtensions.cs b/src/DurableTask.AzureStorage/Linq/AsyncEnumerableExtensions.cs new file mode 100644 index 000000000..3d2301b9a --- /dev/null +++ b/src/DurableTask.AzureStorage/Linq/AsyncEnumerableExtensions.cs @@ -0,0 +1,140 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Linq +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Runtime.CompilerServices; + using System.Threading; + using System.Threading.Tasks; + using Azure; + + static class AsyncEnumerableExtensions + { + // Note: like Enumerable.Select, SelectAsync only supports projecting TSource to TResult + public static async IAsyncEnumerable SelectAsync( + this IEnumerable source, + Func> selectAsync, + [EnumeratorCancellation] CancellationToken cancellationToken = default(CancellationToken)) + { + if (source == null) + { + throw new ArgumentNullException(nameof(source)); + } + + if (selectAsync == null) + { + throw new ArgumentNullException(nameof(selectAsync)); + } + + foreach (TSource item in source) + { + yield return await selectAsync(item, cancellationToken); + } + } + + // Note: The TransformPages methods support potentially complex transformations + // by using delegates that return IEnumerable and IAsyncEnumerable + public static AsyncPageable TransformPages(this AsyncPageable source, Func, IEnumerable> transform) + where TSource : notnull + where TResult : notnull => + new AsyncPageableTransformation(source, transform); + + public static AsyncPageable TransformPagesAsync(this AsyncPageable source, Func, CancellationToken, IAsyncEnumerable> transformAsync) + where TSource : notnull + where TResult : notnull => + new AsyncPageableAsyncTransformation(source, transformAsync); + + sealed class AsyncPageableTransformation : AsyncPageable + where TSource : notnull + where TResult : notnull + { + readonly AsyncPageable source; + readonly Func, IEnumerable> transform; + + public AsyncPageableTransformation(AsyncPageable source, Func, IEnumerable> transform) + { + this.source = source ?? throw new ArgumentNullException(nameof(source)); + this.transform = transform ?? throw new ArgumentNullException(nameof(transform)); + } + + public override IAsyncEnumerable> AsPages(string? continuationToken = null, int? pageSizeHint = null) => + new AsyncEnumerable(this.source.AsPages(continuationToken, pageSizeHint), this.transform); + + sealed class AsyncEnumerable : IAsyncEnumerable> + { + readonly IAsyncEnumerable> source; + readonly Func, IEnumerable> transform; + + public AsyncEnumerable(IAsyncEnumerable> source, Func, IEnumerable> transform) + { + this.source = source; + this.transform = transform; + } + + public async IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default(CancellationToken)) + { + await foreach (Page page in this.source.WithCancellation(cancellationToken)) + { + yield return Page.FromValues( + this.transform(page).ToList(), + page.ContinuationToken, + page.GetRawResponse()); + } + } + } + } + + sealed class AsyncPageableAsyncTransformation : AsyncPageable + where TSource : notnull + where TResult : notnull + { + readonly AsyncPageable source; + readonly Func, CancellationToken, IAsyncEnumerable> transformAsync; + + public AsyncPageableAsyncTransformation(AsyncPageable source, Func, CancellationToken, IAsyncEnumerable> transformAsync) + { + this.source = source ?? throw new ArgumentNullException(nameof(source)); + this.transformAsync = transformAsync ?? throw new ArgumentNullException(nameof(transformAsync)); + } + + public override IAsyncEnumerable> AsPages(string? continuationToken = null, int? pageSizeHint = null) => + new AsyncEnumerable(this.source.AsPages(continuationToken, pageSizeHint), this.transformAsync); + + sealed class AsyncEnumerable : IAsyncEnumerable> + { + readonly IAsyncEnumerable> source; + readonly Func, CancellationToken, IAsyncEnumerable> transformAsync; + + public AsyncEnumerable(IAsyncEnumerable> source, Func, CancellationToken, IAsyncEnumerable> transformAsync) + { + this.source = source; + this.transformAsync = transformAsync; + } + + public async IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default(CancellationToken)) + { + await foreach (Page page in this.source.WithCancellation(cancellationToken)) + { + yield return Page.FromValues( + await this.transformAsync(page, cancellationToken).ToListAsync(cancellationToken), + page.ContinuationToken, + page.GetRawResponse()); + } + } + } + } + } +} diff --git a/src/DurableTask.AzureStorage/Storage/AsyncPageableAsyncProjection.cs b/src/DurableTask.AzureStorage/Storage/AsyncPageableAsyncProjection.cs deleted file mode 100644 index 2a1bf7a7c..000000000 --- a/src/DurableTask.AzureStorage/Storage/AsyncPageableAsyncProjection.cs +++ /dev/null @@ -1,62 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- -#nullable enable -namespace DurableTask.AzureStorage.Storage -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Threading; - using System.Threading.Tasks; - using Azure; - - sealed class AsyncPageableAsyncProjection : AsyncPageable - where TSource : notnull - where TResult : notnull - { - readonly AsyncPageable source; - readonly Func> selector; - - public AsyncPageableAsyncProjection(AsyncPageable source, Func> selector) - { - this.source = source ?? throw new ArgumentNullException(nameof(source)); - this.selector = selector ?? throw new ArgumentNullException(nameof(selector)); - } - - public override IAsyncEnumerable> AsPages(string? continuationToken = null, int? pageSizeHint = null) => - new AsyncPageableProjectionEnumerable(this.source.AsPages(continuationToken, pageSizeHint), this.selector); - - sealed class AsyncPageableProjectionEnumerable : IAsyncEnumerable> - { - readonly IAsyncEnumerable> source; - readonly Func> selector; - - public AsyncPageableProjectionEnumerable(IAsyncEnumerable> source, Func> selector) - { - this.source = source; - this.selector = selector; - } - - public async IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default(CancellationToken)) - { - await foreach (Page page in this.source.WithCancellation(cancellationToken)) - { - yield return Page.FromValues( - await page.Values.ToAsyncEnumerable().SelectAwaitWithCancellation(this.selector).ToListAsync(cancellationToken), - page.ContinuationToken, - page.GetRawResponse()); - } - } - } - } -} diff --git a/src/DurableTask.AzureStorage/Storage/AsyncPageableProjection.cs b/src/DurableTask.AzureStorage/Storage/AsyncPageableProjection.cs deleted file mode 100644 index 57d51df61..000000000 --- a/src/DurableTask.AzureStorage/Storage/AsyncPageableProjection.cs +++ /dev/null @@ -1,59 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- -#nullable enable -namespace DurableTask.AzureStorage.Storage -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Threading; - using System.Threading.Tasks; - using Azure; - - sealed class AsyncPageableProjection : AsyncPageable - where TSource : notnull - where TResult : notnull - { - readonly AsyncPageable source; - readonly Func selector; - - public AsyncPageableProjection(AsyncPageable source, Func selector) - { - this.source = source ?? throw new ArgumentNullException(nameof(source)); - this.selector = selector ?? throw new ArgumentNullException(nameof(selector)); - } - - public override IAsyncEnumerable> AsPages(string? continuationToken = null, int? pageSizeHint = null) => - new AsyncPageableProjectionEnumerable(this.source.AsPages(continuationToken, pageSizeHint), this.selector); - - sealed class AsyncPageableProjectionEnumerable : IAsyncEnumerable> - { - readonly IAsyncEnumerable> source; - readonly Func selector; - - public AsyncPageableProjectionEnumerable(IAsyncEnumerable> source, Func selector) - { - this.source = source; - this.selector = selector; - } - - public async IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default(CancellationToken)) - { - await foreach (Page page in this.source.WithCancellation(cancellationToken)) - { - yield return Page.FromValues(page.Values.Select(this.selector).ToList(), page.ContinuationToken, page.GetRawResponse()); - } - } - } - } -} diff --git a/src/DurableTask.AzureStorage/Storage/BlobContainer.cs b/src/DurableTask.AzureStorage/Storage/BlobContainer.cs index 120394151..4abb9c61c 100644 --- a/src/DurableTask.AzureStorage/Storage/BlobContainer.cs +++ b/src/DurableTask.AzureStorage/Storage/BlobContainer.cs @@ -14,12 +14,14 @@ namespace DurableTask.AzureStorage.Storage { using System; + using System.Linq; using System.Threading; using System.Threading.Tasks; using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; + using DurableTask.AzureStorage.Linq; using DurableTask.AzureStorage.Net; class BlobContainer @@ -68,9 +70,14 @@ public async Task DeleteIfExistsAsync(string? appLeaseId = null, Cancellat public AsyncPageable ListBlobsAsync(string? prefix = null, CancellationToken cancellationToken = default) { - return new AsyncPageableProjection( - this.blobContainerClient.GetBlobsAsync(BlobTraits.Metadata, BlobStates.None, prefix, cancellationToken), - x => this.GetBlobReference(x.Name)).DecorateFailure(); + // GetBlobsAsync will return directories if hierarchical namespace (HNS) is enabled, + // so we must filter them out by checking for a special piece of metadata called "hdi_isfolder" + return this.blobContainerClient + .GetBlobsAsync(BlobTraits.Metadata, BlobStates.None, prefix, cancellationToken) + .DecorateFailure() + .TransformPages(p => p.Values + .Where(b => !IsHnsFolder(b)) + .Select(b => this.GetBlobReference(b.Name))); } public async Task AcquireLeaseAsync(TimeSpan leaseInterval, string leaseId, CancellationToken cancellationToken = default) @@ -100,5 +107,15 @@ public Task RenewLeaseAsync(string leaseId, CancellationToken cancellationToken .RenewAsync(cancellationToken: cancellationToken) .DecorateFailure(); } + + static bool IsHnsFolder(BlobItem item) + { + // Check the optional "hdi_isfolder" value in the metadata to determine whether + // the blob is actually a directory. See https://github.com/Azure/azure-sdk-for-python/issues/24814 + return item.Metadata != null + && item.Metadata.TryGetValue("hdi_isfolder", out string value) + && bool.TryParse(value, out bool isFolder) + && isFolder; + } } } diff --git a/src/DurableTask.AzureStorage/Storage/TableQueryResponse.cs b/src/DurableTask.AzureStorage/Storage/TableQueryResponse.cs index 3c0171093..901a24d30 100644 --- a/src/DurableTask.AzureStorage/Storage/TableQueryResponse.cs +++ b/src/DurableTask.AzureStorage/Storage/TableQueryResponse.cs @@ -21,19 +21,15 @@ namespace DurableTask.AzureStorage.Storage using System.Threading.Tasks; using Azure; - class TableQueryResponse : IAsyncEnumerable where T : notnull + class TableQueryResponse : AsyncPageable where T : notnull { - readonly AsyncPageable _query; + readonly AsyncPageable query; - public TableQueryResponse(AsyncPageable query) - { - this._query = query ?? throw new ArgumentNullException(nameof(query)); - } + public TableQueryResponse(AsyncPageable query) => + this.query = query ?? throw new ArgumentNullException(nameof(query)); - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) - { - return this._query.GetAsyncEnumerator(cancellationToken); - } + public override IAsyncEnumerable> AsPages(string? continuationToken = null, int? pageSizeHint = null) => + this.query.AsPages(continuationToken, pageSizeHint); public async Task> GetResultsAsync(string? continuationToken = null, int? pageSizeHint = null, CancellationToken cancellationToken = default) { @@ -41,7 +37,7 @@ public async Task> GetResultsAsync(string? continuationToke int pages = 0; var entities = new List(); - await foreach (Page page in this._query.AsPages(continuationToken, pageSizeHint).WithCancellation(cancellationToken)) + await foreach (Page page in this.query.AsPages(continuationToken, pageSizeHint).WithCancellation(cancellationToken)) { pages++; entities.AddRange(page.Values); @@ -50,8 +46,5 @@ await foreach (Page page in this._query.AsPages(continuationToken, pageSizeHi sw.Stop(); return new TableQueryResults(entities, sw.Elapsed, pages); } - - public static implicit operator AsyncPageable(TableQueryResponse response) => - response._query; } } diff --git a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs index 282cd50af..4b7e1af7c 100644 --- a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs @@ -27,6 +27,7 @@ namespace DurableTask.AzureStorage.Tracking using System.Threading.Tasks; using Azure; using Azure.Data.Tables; + using DurableTask.AzureStorage.Linq; using DurableTask.AzureStorage.Monitoring; using DurableTask.AzureStorage.Storage; using DurableTask.Core; @@ -542,9 +543,10 @@ public override AsyncPageable GetStateAsync(OrchestrationIns AsyncPageable QueryStateAsync(string filter = null, IEnumerable select = null, CancellationToken cancellationToken = default) { - return new AsyncPageableAsyncProjection( - this.InstancesTable.ExecuteQueryAsync(filter, select: select, cancellationToken: cancellationToken), - (s, t) => new ValueTask(this.ConvertFromAsync(s, KeySanitation.UnescapePartitionKey(s.PartitionKey), t))); + return this.InstancesTable + .ExecuteQueryAsync(filter, select: select, cancellationToken: cancellationToken) + .TransformPagesAsync((p, t) => p.Values + .SelectAsync((s, t) => new ValueTask(this.ConvertFromAsync(s, KeySanitation.UnescapePartitionKey(s.PartitionKey), t)))); } async Task DeleteHistoryAsync( @@ -674,7 +676,7 @@ public override async Task PurgeInstanceHistoryAsync(string CancellationToken cancellationToken = default) { Stopwatch stopwatch = Stopwatch.StartNew(); - List runtimeStatusList = runtimeStatus?.Where( + List runtimeStatusList = runtimeStatus?.Where( status => status == OrchestrationStatus.Completed || status == OrchestrationStatus.Terminated || status == OrchestrationStatus.Canceled || @@ -812,7 +814,7 @@ public override Task StartAsync(CancellationToken cancellationToken = default) int estimatedBytes = 0; IList newEvents = newRuntimeState.NewEvents; IList allEvents = newRuntimeState.Events; - TrackingStoreContext context = (TrackingStoreContext) trackingStoreContext; + TrackingStoreContext context = (TrackingStoreContext)trackingStoreContext; int episodeNumber = Utils.GetEpisodeNumber(newRuntimeState); From 30a41c0d0a51062bb5f1790078a00dfca4cd5687 Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Fri, 5 Jul 2024 12:09:20 -0700 Subject: [PATCH 35/62] Add Extension Method on Queue for Updating MessageData Pop Receipt (#1108) --- .../Messaging/TaskHubQueue.cs | 7 +--- .../Storage/QueueExtensions.cs | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 src/DurableTask.AzureStorage/Storage/QueueExtensions.cs diff --git a/src/DurableTask.AzureStorage/Messaging/TaskHubQueue.cs b/src/DurableTask.AzureStorage/Messaging/TaskHubQueue.cs index d204268bf..230b65e8d 100644 --- a/src/DurableTask.AzureStorage/Messaging/TaskHubQueue.cs +++ b/src/DurableTask.AzureStorage/Messaging/TaskHubQueue.cs @@ -325,13 +325,10 @@ public async Task RenewMessageAsync(MessageData message, SessionBase session) try { - UpdateReceipt receipt = await this.storageQueue.UpdateMessageAsync( - queueMessage, + await this.storageQueue.UpdateMessageAsync( + message, this.MessageVisibilityTimeout, session?.TraceActivityId); - - // Update the pop receipt - message.Update(receipt); } catch (Exception e) { diff --git a/src/DurableTask.AzureStorage/Storage/QueueExtensions.cs b/src/DurableTask.AzureStorage/Storage/QueueExtensions.cs new file mode 100644 index 000000000..27cfbf8c9 --- /dev/null +++ b/src/DurableTask.AzureStorage/Storage/QueueExtensions.cs @@ -0,0 +1,39 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +using System.Threading; +using System; +using System.Threading.Tasks; +using Azure.Storage.Queues.Models; + +namespace DurableTask.AzureStorage.Storage +{ + static class QueueExtensions + { + public static async Task UpdateMessageAsync(this Queue queue, MessageData messageData, TimeSpan visibilityTimeout, Guid? clientRequestId = null, CancellationToken cancellationToken = default) + { + if (queue == null) + { + throw new ArgumentNullException(nameof(queue)); + } + + if (messageData == null) + { + throw new ArgumentNullException(nameof(messageData)); + } + + UpdateReceipt receipt = await queue.UpdateMessageAsync(messageData.OriginalQueueMessage, visibilityTimeout, clientRequestId, cancellationToken); + messageData.Update(receipt); + } + } +} From 9714e8ef8b42c8d6bd384cf44fe8c7d0017001ce Mon Sep 17 00:00:00 2001 From: Varshitha Bachu Date: Thu, 11 Jul 2024 13:37:53 -0700 Subject: [PATCH 36/62] Remove net462 TFM (#1125) --- .../DurableTask.AzureStorage.csproj | 10 +--------- src/DurableTask.Core/DurableTask.Core.csproj | 19 +++++-------------- ...ureTableOrchestrationHistoryEventEntity.cs | 4 ---- 3 files changed, 6 insertions(+), 27 deletions(-) diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 81aa6bae6..19d08a87e 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -2,7 +2,7 @@ - netstandard2.0;net462 + netstandard2.0 true true Azure Storage provider extension for the Durable Task Framework. @@ -46,14 +46,6 @@ - - - - - - - - diff --git a/src/DurableTask.Core/DurableTask.Core.csproj b/src/DurableTask.Core/DurableTask.Core.csproj index 25c247e1b..786e1e42c 100644 --- a/src/DurableTask.Core/DurableTask.Core.csproj +++ b/src/DurableTask.Core/DurableTask.Core.csproj @@ -2,7 +2,7 @@ - netstandard2.0;net462 + netstandard2.0 true NU5125;NU5048;CS7035 @@ -16,9 +16,9 @@ - 2 - 17 - 1 + 3 + 0 + 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) $(VersionPrefix).0 @@ -34,19 +34,10 @@ $(VersionPrefix)-$(VersionSuffix) - - - - - - - + - - - diff --git a/src/DurableTask.ServiceBus/Tracking/AzureTableOrchestrationHistoryEventEntity.cs b/src/DurableTask.ServiceBus/Tracking/AzureTableOrchestrationHistoryEventEntity.cs index f9a098137..99272b67d 100644 --- a/src/DurableTask.ServiceBus/Tracking/AzureTableOrchestrationHistoryEventEntity.cs +++ b/src/DurableTask.ServiceBus/Tracking/AzureTableOrchestrationHistoryEventEntity.cs @@ -35,11 +35,7 @@ internal class AzureTableOrchestrationHistoryEventEntity : AzureTableCompositeTa private static readonly JsonSerializerSettings ReadJsonSettings = new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Objects, -#if NETSTANDARD2_0 SerializationBinder = new PackageUpgradeSerializationBinder() -#else - Binder = new PackageUpgradeSerializationBinder() -#endif }; /// From 0c887ce1cd869bc188da632750ee806cc3c9584e Mon Sep 17 00:00:00 2001 From: Varshitha Bachu Date: Thu, 11 Jul 2024 15:00:29 -0700 Subject: [PATCH 37/62] Update validate-build.yml to run DT.AS unit tests and change default value of UseTablePartitionManagement to true (#1133) --- .github/workflows/validate-build.yml | 5 ++--- .../AzureStorageOrchestrationServiceSettings.cs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate-build.yml b/.github/workflows/validate-build.yml index 0882c501d..3e55c0595 100644 --- a/.github/workflows/validate-build.yml +++ b/.github/workflows/validate-build.yml @@ -53,7 +53,6 @@ jobs: - name: Test DTFx.Core run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj #--configuration $config --no-build --verbosity normal - # Azure Storage is commented out until DTFx.AS v2 is enabled, where Azurite can be used to run unit tests - # - name: Test DTFx.AzureStorage - # run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj #--configuration $config --no-build --verbosity normal + - name: Test DTFx.AzureStorage + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj #--configuration $config --no-build --verbosity normal diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs index 3af54ab89..71121daf8 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs @@ -193,7 +193,7 @@ public class AzureStorageOrchestrationServiceSettings /// /// Use the newer Azure Tables-based partition manager instead of the older Azure Blobs-based partition manager. The default value is false. /// - public bool UseTablePartitionManagement { get; set; } = false; + public bool UseTablePartitionManagement { get; set; } = true; /// /// User serialization that will respect . Default is false. From b3fb3b6fb2bc7addde7abf3bc2dc426fe678d7c9 Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 11 Jul 2024 16:13:45 -0700 Subject: [PATCH 38/62] skip api check in Azurite --- .github/workflows/validate-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-build.yml b/.github/workflows/validate-build.yml index 3e55c0595..857b6eb43 100644 --- a/.github/workflows/validate-build.yml +++ b/.github/workflows/validate-build.yml @@ -54,5 +54,5 @@ jobs: run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj #--configuration $config --no-build --verbosity normal - name: Test DTFx.AzureStorage - run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj #--configuration $config --no-build --verbosity normal + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj #--configuration $config --no-build --verbosity normal From 9c996da083a5734acf79cb93913889d8c05fd3f3 Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 11 Jul 2024 18:24:21 -0700 Subject: [PATCH 39/62] split test burden --- .github/workflows/validate-build-2.yml | 53 ++++++++++++++++++++++++++ .github/workflows/validate-build-3.yml | 53 ++++++++++++++++++++++++++ .github/workflows/validate-build-4.yml | 53 ++++++++++++++++++++++++++ .github/workflows/validate-build.yml | 6 +-- 4 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/validate-build-2.yml create mode 100644 .github/workflows/validate-build-3.yml create mode 100644 .github/workflows/validate-build-4.yml diff --git a/.github/workflows/validate-build-2.yml b/.github/workflows/validate-build-2.yml new file mode 100644 index 000000000..1e1504451 --- /dev/null +++ b/.github/workflows/validate-build-2.yml @@ -0,0 +1,53 @@ +name: Validate Build (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) + +on: + push: + branches: + - main + paths-ignore: [ '**.md' ] + pull_request: + branches: + - main + paths-ignore: [ '**.md' ] + +env: + solution: DurableTask.sln + config: Release + +jobs: + build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + + - name: Set up .NET Core 2.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '2.1.x' + + - name: Set up .NET Core 3.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.x' + + - name: Restore dependencies + run: dotnet restore $solution + + - name: Build + run: dotnet build $solution #--configuration $config #--no-restore -p:FileVersionRevision=$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=true + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: '16.x' + + - name: Install Azurite + run: npm install -g azurite + - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.* !FullyQualifiedName~DurableTask.AzureStorage.Tests.ExcludeFile1!* !FullyQualifiedName~DurableTask.AzureStorage.Tests.ExcludeFile2!*" \ No newline at end of file diff --git a/.github/workflows/validate-build-3.yml b/.github/workflows/validate-build-3.yml new file mode 100644 index 000000000..f07b85a04 --- /dev/null +++ b/.github/workflows/validate-build-3.yml @@ -0,0 +1,53 @@ +name: Validate Build (Only AzureStorageScaleTests) + +on: + push: + branches: + - main + paths-ignore: [ '**.md' ] + pull_request: + branches: + - main + paths-ignore: [ '**.md' ] + +env: + solution: DurableTask.sln + config: Release + +jobs: + build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + + - name: Set up .NET Core 2.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '2.1.x' + + - name: Set up .NET Core 3.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.x' + + - name: Restore dependencies + run: dotnet restore $solution + + - name: Build + run: dotnet build $solution #--configuration $config #--no-restore -p:FileVersionRevision=$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=true + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: '16.x' + + - name: Install Azurite + run: npm install -g azurite + - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScaleTests" diff --git a/.github/workflows/validate-build-4.yml b/.github/workflows/validate-build-4.yml new file mode 100644 index 000000000..d96a1680e --- /dev/null +++ b/.github/workflows/validate-build-4.yml @@ -0,0 +1,53 @@ +name: Validate Build (Only AzureStorageScenarioTests) + +on: + push: + branches: + - main + paths-ignore: [ '**.md' ] + pull_request: + branches: + - main + paths-ignore: [ '**.md' ] + +env: + solution: DurableTask.sln + config: Release + +jobs: + build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + + - name: Set up .NET Core 2.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '2.1.x' + + - name: Set up .NET Core 3.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.x' + + - name: Restore dependencies + run: dotnet restore $solution + + - name: Build + run: dotnet build $solution #--configuration $config #--no-restore -p:FileVersionRevision=$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=true + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: '16.x' + + - name: Install Azurite + run: npm install -g azurite + - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests*" diff --git a/.github/workflows/validate-build.yml b/.github/workflows/validate-build.yml index 857b6eb43..bfff1f8ca 100644 --- a/.github/workflows/validate-build.yml +++ b/.github/workflows/validate-build.yml @@ -1,4 +1,4 @@ -name: Validate Build (DTFx.Core) +name: Validate Build (DTFx.Core and basic DTFx.AS) on: push: @@ -53,6 +53,6 @@ jobs: - name: Test DTFx.Core run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj #--configuration $config --no-build --verbosity normal - - name: Test DTFx.AzureStorage - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj #--configuration $config --no-build --verbosity normal + - name: Test DTFx.AzureStorage (except DurableTask.AzureStorage.Tests) + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName!~DurableTask.AzureStorage.Tests*" #--configuration $config --no-build --verbosity normal From 9e2f2bcd7697de9286a3cf3aecacf2d459a20de2 Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 11 Jul 2024 19:58:05 -0700 Subject: [PATCH 40/62] fix typo in github actions exclusion of tests --- .github/workflows/validate-build-2.yml | 2 +- .github/workflows/validate-build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-build-2.yml b/.github/workflows/validate-build-2.yml index 1e1504451..aee1ee0db 100644 --- a/.github/workflows/validate-build-2.yml +++ b/.github/workflows/validate-build-2.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.* !FullyQualifiedName~DurableTask.AzureStorage.Tests.ExcludeFile1!* !FullyQualifiedName~DurableTask.AzureStorage.Tests.ExcludeFile2!*" \ No newline at end of file + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.* !FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScaleTests!* !FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests!*" \ No newline at end of file diff --git a/.github/workflows/validate-build.yml b/.github/workflows/validate-build.yml index bfff1f8ca..7b55f3136 100644 --- a/.github/workflows/validate-build.yml +++ b/.github/workflows/validate-build.yml @@ -54,5 +54,5 @@ jobs: run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj #--configuration $config --no-build --verbosity normal - name: Test DTFx.AzureStorage (except DurableTask.AzureStorage.Tests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName!~DurableTask.AzureStorage.Tests*" #--configuration $config --no-build --verbosity normal + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName!~DurableTask.AzureStorage.Tests.*" #--configuration $config --no-build --verbosity normal From f80f3a38d105b12112db15f41f064ebe4fd997d2 Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 11 Jul 2024 20:08:30 -0700 Subject: [PATCH 41/62] fix filtering syntax --- .github/workflows/validate-build-2.yml | 2 +- .github/workflows/validate-build-3.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-build-2.yml b/.github/workflows/validate-build-2.yml index aee1ee0db..eee0bfbdd 100644 --- a/.github/workflows/validate-build-2.yml +++ b/.github/workflows/validate-build-2.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.* !FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScaleTests!* !FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests!*" \ No newline at end of file + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.* | FullyQualifiedName!~DurableTask.AzureStorage.Tests.AzureStorageScaleTests* | FullyQualifiedName!~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests*" \ No newline at end of file diff --git a/.github/workflows/validate-build-3.yml b/.github/workflows/validate-build-3.yml index f07b85a04..0f50bb2bd 100644 --- a/.github/workflows/validate-build-3.yml +++ b/.github/workflows/validate-build-3.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScaleTests" + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScaleTests.*" From 6bfed97b340a6293cabd6d5787caae448087bbc4 Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 11 Jul 2024 20:10:32 -0700 Subject: [PATCH 42/62] use logical and --- .github/workflows/validate-build-2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-build-2.yml b/.github/workflows/validate-build-2.yml index eee0bfbdd..410b848cd 100644 --- a/.github/workflows/validate-build-2.yml +++ b/.github/workflows/validate-build-2.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.* | FullyQualifiedName!~DurableTask.AzureStorage.Tests.AzureStorageScaleTests* | FullyQualifiedName!~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests*" \ No newline at end of file + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.* & FullyQualifiedName!~DurableTask.AzureStorage.Tests.AzureStorageScaleTests* & FullyQualifiedName!~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests*" \ No newline at end of file From ce3ff4cdf5d15f9b48fcf24c700a469f57edc957 Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 11 Jul 2024 20:57:01 -0700 Subject: [PATCH 43/62] simplify syntax --- .github/workflows/validate-build-2.yml | 2 +- .github/workflows/validate-build-3.yml | 2 +- .github/workflows/validate-build-4.yml | 2 +- .github/workflows/validate-build.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate-build-2.yml b/.github/workflows/validate-build-2.yml index 410b848cd..7a63c016a 100644 --- a/.github/workflows/validate-build-2.yml +++ b/.github/workflows/validate-build-2.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.* & FullyQualifiedName!~DurableTask.AzureStorage.Tests.AzureStorageScaleTests* & FullyQualifiedName!~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests*" \ No newline at end of file + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests & FullyQualifiedName!~DurableTask.AzureStorage.Tests.AzureStorageScaleTests & FullyQualifiedName!~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests" \ No newline at end of file diff --git a/.github/workflows/validate-build-3.yml b/.github/workflows/validate-build-3.yml index 0f50bb2bd..f07b85a04 100644 --- a/.github/workflows/validate-build-3.yml +++ b/.github/workflows/validate-build-3.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScaleTests.*" + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScaleTests" diff --git a/.github/workflows/validate-build-4.yml b/.github/workflows/validate-build-4.yml index d96a1680e..c3f0d458a 100644 --- a/.github/workflows/validate-build-4.yml +++ b/.github/workflows/validate-build-4.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests*" + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests" diff --git a/.github/workflows/validate-build.yml b/.github/workflows/validate-build.yml index 7b55f3136..65637c3b9 100644 --- a/.github/workflows/validate-build.yml +++ b/.github/workflows/validate-build.yml @@ -54,5 +54,5 @@ jobs: run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj #--configuration $config --no-build --verbosity normal - name: Test DTFx.AzureStorage (except DurableTask.AzureStorage.Tests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName!~DurableTask.AzureStorage.Tests.*" #--configuration $config --no-build --verbosity normal + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName!~DurableTask.AzureStorage.Tests" #--configuration $config --no-build --verbosity normal From 8b775cf2f5cd58efd731ccdfdf6d7f8f0dcf707a Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 11 Jul 2024 21:30:22 -0700 Subject: [PATCH 44/62] simplify actions --- .github/workflows/validate-build-2.yml | 2 +- .github/workflows/validate-build.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate-build-2.yml b/.github/workflows/validate-build-2.yml index 7a63c016a..e29270654 100644 --- a/.github/workflows/validate-build-2.yml +++ b/.github/workflows/validate-build-2.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests & FullyQualifiedName!~DurableTask.AzureStorage.Tests.AzureStorageScaleTests & FullyQualifiedName!~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests" \ No newline at end of file + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests & FullyQualifiedName!~AzureStorageScaleTests & FullyQualifiedName!~AzureStorageScenarioTests" \ No newline at end of file diff --git a/.github/workflows/validate-build.yml b/.github/workflows/validate-build.yml index 65637c3b9..f36b0bd2f 100644 --- a/.github/workflows/validate-build.yml +++ b/.github/workflows/validate-build.yml @@ -53,6 +53,6 @@ jobs: - name: Test DTFx.Core run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj #--configuration $config --no-build --verbosity normal - - name: Test DTFx.AzureStorage (except DurableTask.AzureStorage.Tests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName!~DurableTask.AzureStorage.Tests" #--configuration $config --no-build --verbosity normal + - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Net tests) + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Net" #--configuration $config --no-build --verbosity normal From cb1b6a28ed6a42fdc1fc324ebfab51d7840e6858 Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 11 Jul 2024 21:36:55 -0700 Subject: [PATCH 45/62] leverage classname --- .github/workflows/validate-build-2.yml | 2 +- .github/workflows/validate-build-4.yml | 2 +- .github/workflows/validate-build.yml | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate-build-2.yml b/.github/workflows/validate-build-2.yml index e29270654..07055bef6 100644 --- a/.github/workflows/validate-build-2.yml +++ b/.github/workflows/validate-build-2.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests & FullyQualifiedName!~AzureStorageScaleTests & FullyQualifiedName!~AzureStorageScenarioTests" \ No newline at end of file + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests & ClassName!=AzureStorageScaleTests & ClassName!=AzureStorageScenarioTests" \ No newline at end of file diff --git a/.github/workflows/validate-build-4.yml b/.github/workflows/validate-build-4.yml index c3f0d458a..9b7ea7bf2 100644 --- a/.github/workflows/validate-build-4.yml +++ b/.github/workflows/validate-build-4.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests" + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests" \ No newline at end of file diff --git a/.github/workflows/validate-build.yml b/.github/workflows/validate-build.yml index f36b0bd2f..d259decb9 100644 --- a/.github/workflows/validate-build.yml +++ b/.github/workflows/validate-build.yml @@ -54,5 +54,4 @@ jobs: run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj #--configuration $config --no-build --verbosity normal - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Net tests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Net" #--configuration $config --no-build --verbosity normal - + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Net" #--configuration $config --no-build --verbosity normal \ No newline at end of file From 85f69841f8b548230fce6a07cb7f53cc22d4de5a Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 11 Jul 2024 22:16:52 -0700 Subject: [PATCH 46/62] change to using classname --- .github/workflows/validate-build-2.yml | 2 +- .github/workflows/validate-build-3.yml | 4 ++-- .github/workflows/validate-build-4.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate-build-2.yml b/.github/workflows/validate-build-2.yml index 07055bef6..f69b49870 100644 --- a/.github/workflows/validate-build-2.yml +++ b/.github/workflows/validate-build-2.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests & ClassName!=AzureStorageScaleTests & ClassName!=AzureStorageScenarioTests" \ No newline at end of file + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests & ClassName!=DurableTask.AzureStorage.AzureStorageScaleTests & ClassName!=DurableTask.AzureStorage.AzureStorageScenarioTests" \ No newline at end of file diff --git a/.github/workflows/validate-build-3.yml b/.github/workflows/validate-build-3.yml index f07b85a04..887287407 100644 --- a/.github/workflows/validate-build-3.yml +++ b/.github/workflows/validate-build-3.yml @@ -49,5 +49,5 @@ jobs: - name: Install Azurite run: npm install -g azurite - - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScaleTests" + - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTestss) + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "ClassName=DurableTask.AzureStorage.Tests.AzureStorageScaleTests" diff --git a/.github/workflows/validate-build-4.yml b/.github/workflows/validate-build-4.yml index 9b7ea7bf2..b11f8916e 100644 --- a/.github/workflows/validate-build-4.yml +++ b/.github/workflows/validate-build-4.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests.AzureStorageScenarioTests" \ No newline at end of file + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "ClassName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests" \ No newline at end of file From cd26939685abeb345ebfcf6092c0eb6195e6c4e1 Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 11 Jul 2024 22:25:58 -0700 Subject: [PATCH 47/62] add missing tests element --- .github/workflows/validate-build-2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-build-2.yml b/.github/workflows/validate-build-2.yml index f69b49870..1f0bc4a38 100644 --- a/.github/workflows/validate-build-2.yml +++ b/.github/workflows/validate-build-2.yml @@ -50,4 +50,4 @@ jobs: - name: Install Azurite run: npm install -g azurite - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests & ClassName!=DurableTask.AzureStorage.AzureStorageScaleTests & ClassName!=DurableTask.AzureStorage.AzureStorageScenarioTests" \ No newline at end of file + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests & ClassName!=DurableTask.AzureStorage.Tests.AzureStorageScaleTests & ClassName!=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests" \ No newline at end of file From a8ed5bed313915bfce3bdb8d9af1e4a5cb19a6cc Mon Sep 17 00:00:00 2001 From: David Justo Date: Fri, 12 Jul 2024 10:40:25 -0700 Subject: [PATCH 48/62] spread out tests --- .github/workflows/validate-build-3.yml | 2 +- .github/workflows/validate-build-4.yml | 7 ++-- .github/workflows/validate-build-5.yml | 54 ++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/validate-build-5.yml diff --git a/.github/workflows/validate-build-3.yml b/.github/workflows/validate-build-3.yml index 887287407..80d72c8af 100644 --- a/.github/workflows/validate-build-3.yml +++ b/.github/workflows/validate-build-3.yml @@ -49,5 +49,5 @@ jobs: - name: Install Azurite run: npm install -g azurite - - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTestss) + - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests) run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "ClassName=DurableTask.AzureStorage.Tests.AzureStorageScaleTests" diff --git a/.github/workflows/validate-build-4.yml b/.github/workflows/validate-build-4.yml index b11f8916e..64ca1d9a1 100644 --- a/.github/workflows/validate-build-4.yml +++ b/.github/workflows/validate-build-4.yml @@ -1,4 +1,4 @@ -name: Validate Build (Only AzureStorageScenarioTests) +name: Validate Build (AzureStorageScenarioTests 1/2) on: push: @@ -49,5 +49,6 @@ jobs: - name: Install Azurite run: npm install -g azurite - - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "ClassName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests" \ No newline at end of file + + - name: Test DTFx.AzureStorage (AzureStorageScenarioTests 1/2) + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "ClassName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests & FullyQualifiedName!=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.LargeTableTextMessagePayloads_SizeViolation_BlobUrl & FullyQualifiedName!=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.LargeTableTextMessagePayloads_FetchLargeMessages & FullyQualifiedName!=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.ScheduledStart_Inline & FullyQualifiedName!=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.ScheduledStart_Activity" \ No newline at end of file diff --git a/.github/workflows/validate-build-5.yml b/.github/workflows/validate-build-5.yml new file mode 100644 index 000000000..88906a690 --- /dev/null +++ b/.github/workflows/validate-build-5.yml @@ -0,0 +1,54 @@ +name: Validate Build (AzureStorageScenarioTests 2/2) + +on: + push: + branches: + - main + paths-ignore: [ '**.md' ] + pull_request: + branches: + - main + paths-ignore: [ '**.md' ] + +env: + solution: DurableTask.sln + config: Release + +jobs: + build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + + - name: Set up .NET Core 2.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '2.1.x' + + - name: Set up .NET Core 3.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.x' + + - name: Restore dependencies + run: dotnet restore $solution + + - name: Build + run: dotnet build $solution #--configuration $config #--no-restore -p:FileVersionRevision=$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=true + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: '16.x' + + - name: Install Azurite + run: npm install -g azurite + + - name: Test DTFx.AzureStorage (AzureStorageScenarioTests 1/2) + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.LargeTableTextMessagePayloads_SizeViolation_BlobUrl & FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.LargeTableTextMessagePayloads_FetchLargeMessages & FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.ScheduledStart_Inline & FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.ScheduledStart_Activity" \ No newline at end of file From 2be78a2d10d65c47951f14c039d7cfd1ccc03fbb Mon Sep 17 00:00:00 2001 From: David Justo Date: Fri, 12 Jul 2024 11:00:02 -0700 Subject: [PATCH 49/62] fix filter --- .github/workflows/validate-build-5.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-build-5.yml b/.github/workflows/validate-build-5.yml index 88906a690..65adc228f 100644 --- a/.github/workflows/validate-build-5.yml +++ b/.github/workflows/validate-build-5.yml @@ -50,5 +50,5 @@ jobs: - name: Install Azurite run: npm install -g azurite - - name: Test DTFx.AzureStorage (AzureStorageScenarioTests 1/2) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.LargeTableTextMessagePayloads_SizeViolation_BlobUrl & FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.LargeTableTextMessagePayloads_FetchLargeMessages & FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.ScheduledStart_Inline & FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.ScheduledStart_Activity" \ No newline at end of file + - name: Test DTFx.AzureStorage (AzureStorageScenarioTests 2/2) + run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 | dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.LargeTableTextMessagePayloads_SizeViolation_BlobUrl | FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.LargeTableTextMessagePayloads_FetchLargeMessages | FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.ScheduledStart_Inline | FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.ScheduledStart_Activity" \ No newline at end of file From c92bd248dafa1eae79e253549d0b1f1ef8decdeb Mon Sep 17 00:00:00 2001 From: David Justo Date: Fri, 12 Jul 2024 18:14:52 -0700 Subject: [PATCH 50/62] add CI --- eng/ci/public-build.yml | 68 +++++++++++++++++++++ eng/templates/build-steps.yml | 107 ++++++++++++++++++++++++++++++++++ eng/templates/test.yml | 31 ++++++++++ 3 files changed, 206 insertions(+) create mode 100644 eng/ci/public-build.yml create mode 100644 eng/templates/build-steps.yml create mode 100644 eng/templates/test.yml diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml new file mode 100644 index 000000000..68d01ae51 --- /dev/null +++ b/eng/ci/public-build.yml @@ -0,0 +1,68 @@ +# This pipeline is used for public PR and CI builds. + +# Run on changes in main +trigger: + batch: true + branches: + include: + - main + +# Run nightly to catch new CVEs and to report SDL often. +schedules: + - cron: "0 0 * * *" + displayName: Nightly Run + branches: + include: + - main + always: true # Run pipeline irrespective of no code changes since last successful run + +# Run on all PRs +pr: + branches: + include: + - '*' + +# This allows us to reference 1ES templates, our pipelines extend a pre-existing template +resources: + repositories: + - repository: 1es + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + # The template we extend injects compliance-checks into the pipleine, such as SDL and CodeQL + template: v1/1ES.Unofficial.PipelineTemplate.yml@1es + parameters: + pool: + name: 1es-pool-azfunc-public + image: 1es-windows-2022 + os: windows + + sdl: + codeql: + compiled: + enabled: true + runSourceLanguagesInSourceAnalysis: true + + settings: + # PR's from forks should not have permissions to set tags. + skipBuildTagsForGitHubPullRequests: ${{ variables['System.PullRequest.IsFork'] }} + + stages: + - stage: Validate + jobs: + - job: Validate + strategy: + parallel: 13 + steps: + # Build the code and the tests + - template: /eng/templates/build-steps.yml@self + parameters: + # The tests only build in the 'Debug' configuration. + # In the release configuration, the packages don't expose their internals + # to the test projects. + buildConfiguration: 'Debug' + buildTests: true + # Run tests + - template: /eng/templates/test.yml@self \ No newline at end of file diff --git a/eng/templates/build-steps.yml b/eng/templates/build-steps.yml new file mode 100644 index 000000000..6cd1468d7 --- /dev/null +++ b/eng/templates/build-steps.yml @@ -0,0 +1,107 @@ +parameters: +- name: buildConfiguration + type: string + default: 'Release' + +- name: buildTests + type: boolean + default: false + +steps: +# Start by restoring all the dependencies. This needs to be its own task +# from what I can tell. We specifically only target DurableTask.AzureStorage +# and its direct dependencies. +# Configure all the .NET SDK versions we need +- task: UseDotNet@2 + displayName: 'Use the .NET Core 2.1 SDK (required for build signing)' + inputs: + packageType: 'sdk' + version: '2.1.x' + +- task: UseDotNet@2 + displayName: 'Use the .NET Core 3.1 SDK' + inputs: + packageType: 'sdk' + version: '3.1.x' + +- task: UseDotNet@2 + displayName: 'Use the .NET 6 SDK' + inputs: + packageType: 'sdk' + version: '6.0.x' + +- task: DotNetCoreCLI@2 + displayName: 'Restore nuget dependencies' + inputs: + command: restore + verbosityRestore: Minimal + projects: | + src/DurableTask.AzureStorage/DurableTask.AzureStorage.sln + src/DurableTask.Emulator/DurableTask.Emulator.csproj + src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj + +# Build the filtered solution in release mode, specifying the continuous integration flag. +- task: VSBuild@1 + displayName: 'Build (AzureStorage)' + inputs: + solution: 'src/DurableTask.AzureStorage/DurableTask.AzureStorage.sln' + vsVersion: '17.0' + logFileVerbosity: minimal + configuration: ${{ parameters.buildConfiguration }} + msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true + +- task: VSBuild@1 + displayName: 'Build (ApplicationInsights)' + inputs: + solution: 'src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj' + vsVersion: '17.0' + logFileVerbosity: minimal + configuration: ${{ parameters.buildConfiguration }} + msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true + +- task: VSBuild@1 + displayName: 'Build (Emulator)' + inputs: + solution: 'src/DurableTask.Emulator/DurableTask.Emulator.csproj' + vsVersion: '17.0' + logFileVerbosity: minimal + configuration: ${{ parameters.buildConfiguration }} + msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true + +- ${{ if eq(parameters.buildTests, true) }}: + - task: DotNetCoreCLI@2 + displayName: 'Restore nuget dependencies' + inputs: + command: restore + verbosityRestore: Minimal + projects: | + .\Test\DurableTask.Core.Tests\DurableTask.Core.Tests.csproj + .\Test\DurableTask.AzureStorage.Tests\DurableTask.AzureStorage.Tests.csproj + .\Test\DurableTask.Emulator.Tests\DurableTask.Emulator.Tests.csproj + + - task: VSBuild@1 + displayName: 'Build (Core Tests)' + inputs: + solution: '.\Test\DurableTask.Core.Tests\DurableTask.Core.Tests.csproj' + vsVersion: '17.0' + logFileVerbosity: minimal + configuration: ${{ parameters.buildConfiguration }} + msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true + + - task: VSBuild@1 + displayName: 'Build (AzureStorage Tests)' + inputs: + solution: '.\Test\DurableTask.AzureStorage.Tests\DurableTask.AzureStorage.Tests.csproj' + vsVersion: '17.0' + logFileVerbosity: minimal + configuration: ${{ parameters.buildConfiguration }} + msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true + + - task: VSBuild@1 + displayName: 'Build (Emulator Tests)' + inputs: + solution: '.\Test\DurableTask.Emulator.Tests\DurableTask.Emulator.Tests.csproj' + vsVersion: '17.0' + logFileVerbosity: minimal + configuration: ${{ parameters.buildConfiguration }} + msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true \ No newline at end of file diff --git a/eng/templates/test.yml b/eng/templates/test.yml new file mode 100644 index 000000000..1d04822a3 --- /dev/null +++ b/eng/templates/test.yml @@ -0,0 +1,31 @@ +steps: + # Install Azurite + - bash: | + echo "Installing azurite" + npm install -g azurite + mkdir azurite1 + echo "azurite installed" + azurite --silent --location azurite1 --debug azurite1\debug.txt --queuePort 10001 & + echo "azurite started" + sleep 5 + displayName: 'Install and Run Azurite' + + # Run tests + - task: VSTest@2 + displayName: 'Run tests' + inputs: + testAssemblyVer2: | + **\bin\**\DurableTask.AzureStorage.Tests.dll + **\bin\**\DurableTask.Core.Tests.dll + **\bin\**\DurableTask.Emulator.Tests.dll + !**\obj\** + testFiltercriteria: 'TestCategory!=DisabledInCI' + vsTestVersion: 16.0 + distributionBatchType: basedOnExecutionTime + platform: 'any cpu' + configuration: 'Debug' + diagnosticsEnabled: True + collectDumpOn: always + rerunFailedTests: true + rerunFailedThreshold: 20 + rerunMaxAttempts: 2 \ No newline at end of file From d50473a7b31f7c1f2ee3600cd1b30dbb89c5aea4 Mon Sep 17 00:00:00 2001 From: David Justo Date: Fri, 12 Jul 2024 23:40:49 -0700 Subject: [PATCH 51/62] tweaks --- eng/templates/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/templates/test.yml b/eng/templates/test.yml index 1d04822a3..a36079c99 100644 --- a/eng/templates/test.yml +++ b/eng/templates/test.yml @@ -15,12 +15,12 @@ steps: displayName: 'Run tests' inputs: testAssemblyVer2: | - **\bin\**\DurableTask.AzureStorage.Tests.dll - **\bin\**\DurableTask.Core.Tests.dll - **\bin\**\DurableTask.Emulator.Tests.dll + **\bin\net462\DurableTask.AzureStorage.Tests.dll + **\bin\net462\DurableTask.Core.Tests.dll + **\bin\net462\DurableTask.Emulator.Tests.dll !**\obj\** testFiltercriteria: 'TestCategory!=DisabledInCI' - vsTestVersion: 16.0 + vsTestVersion: 17.0 distributionBatchType: basedOnExecutionTime platform: 'any cpu' configuration: 'Debug' From eb9f402bb431376daa0c85404bcd164eb19664d8 Mon Sep 17 00:00:00 2001 From: David Justo Date: Fri, 12 Jul 2024 23:54:28 -0700 Subject: [PATCH 52/62] increase retry capabilities --- eng/templates/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/templates/test.yml b/eng/templates/test.yml index a36079c99..7d8849bfa 100644 --- a/eng/templates/test.yml +++ b/eng/templates/test.yml @@ -27,5 +27,5 @@ steps: diagnosticsEnabled: True collectDumpOn: always rerunFailedTests: true - rerunFailedThreshold: 20 - rerunMaxAttempts: 2 \ No newline at end of file + rerunFailedThreshold: 30 + rerunMaxAttempts: 3 \ No newline at end of file From c13da7b82f53549bb627cdcc207e9ae1d113eace Mon Sep 17 00:00:00 2001 From: David Justo Date: Sat, 13 Jul 2024 00:02:12 -0700 Subject: [PATCH 53/62] change tfm --- eng/templates/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/templates/test.yml b/eng/templates/test.yml index 7d8849bfa..2d69212af 100644 --- a/eng/templates/test.yml +++ b/eng/templates/test.yml @@ -15,9 +15,9 @@ steps: displayName: 'Run tests' inputs: testAssemblyVer2: | - **\bin\net462\DurableTask.AzureStorage.Tests.dll - **\bin\net462\DurableTask.Core.Tests.dll - **\bin\net462\DurableTask.Emulator.Tests.dll + **\bin\netcoreapp3.1\DurableTask.AzureStorage.Tests.dll + **\bin\netcoreapp3.1\DurableTask.Core.Tests.dll + **\bin\netcoreapp3.1\DurableTask.Emulator.Tests.dll !**\obj\** testFiltercriteria: 'TestCategory!=DisabledInCI' vsTestVersion: 17.0 From 18c7ecaa82369f88466a477f1da7001be42b2fae Mon Sep 17 00:00:00 2001 From: David Justo Date: Sat, 13 Jul 2024 00:19:37 -0700 Subject: [PATCH 54/62] fix test paths --- eng/templates/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/templates/test.yml b/eng/templates/test.yml index 2d69212af..4771c5d51 100644 --- a/eng/templates/test.yml +++ b/eng/templates/test.yml @@ -15,9 +15,9 @@ steps: displayName: 'Run tests' inputs: testAssemblyVer2: | - **\bin\netcoreapp3.1\DurableTask.AzureStorage.Tests.dll - **\bin\netcoreapp3.1\DurableTask.Core.Tests.dll - **\bin\netcoreapp3.1\DurableTask.Emulator.Tests.dll + **\bin\**\netcoreapp3.1\DurableTask.AzureStorage.Tests.dll + **\bin\**\netcoreapp3.1\DurableTask.Core.Tests.dll + **\bin\**\netcoreapp3.1\DurableTask.Emulator.Tests.dll !**\obj\** testFiltercriteria: 'TestCategory!=DisabledInCI' vsTestVersion: 17.0 From b5e3157ed82556755d7776eddd0b8a91a9411a75 Mon Sep 17 00:00:00 2001 From: David Justo Date: Sat, 13 Jul 2024 01:06:00 -0700 Subject: [PATCH 55/62] refactor into stages --- eng/ci/public-build.yml | 42 +++++++++++++++++++++++++++++++++++++++-- eng/templates/test.yml | 10 +++++++--- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index 68d01ae51..2292aa19e 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -50,7 +50,7 @@ extends: skipBuildTagsForGitHubPullRequests: ${{ variables['System.PullRequest.IsFork'] }} stages: - - stage: Validate + - stage: DTFx.CoreValidate jobs: - job: Validate strategy: @@ -65,4 +65,42 @@ extends: buildConfiguration: 'Debug' buildTests: true # Run tests - - template: /eng/templates/test.yml@self \ No newline at end of file + - template: /eng/templates/test.yml@self + parameters: + testAssembly: '**\bin\**\netcoreapp3.1\DurableTask.Core.Tests.dll' + - stage: DTFx.ASValidate + jobs: + - job: Validate + strategy: + parallel: 13 + steps: + # Build the code and the tests + - template: /eng/templates/build-steps.yml@self + parameters: + # The tests only build in the 'Debug' configuration. + # In the release configuration, the packages don't expose their internals + # to the test projects. + buildConfiguration: 'Debug' + buildTests: true + # Run tests + - template: /eng/templates/test.yml@self + parameters: + testAssembly: '**\bin\**\netcoreapp3.1\DurableTask.AzureStorage.Tests.dll' + - stage: DTFx.EmulatorValidate + jobs: + - job: Validate + strategy: + parallel: 13 + steps: + # Build the code and the tests + - template: /eng/templates/build-steps.yml@self + parameters: + # The tests only build in the 'Debug' configuration. + # In the release configuration, the packages don't expose their internals + # to the test projects. + buildConfiguration: 'Debug' + buildTests: true + # Run tests + - template: /eng/templates/test.yml@self + parameters: + testAssembly: '**\bin\**\netcoreapp3.1\DurableTask.Emulator.Tests.dll' \ No newline at end of file diff --git a/eng/templates/test.yml b/eng/templates/test.yml index 4771c5d51..2cd5615f9 100644 --- a/eng/templates/test.yml +++ b/eng/templates/test.yml @@ -1,3 +1,9 @@ +parameters: +- name: testAssembly + type: string + default: '' + + steps: # Install Azurite - bash: | @@ -15,9 +21,7 @@ steps: displayName: 'Run tests' inputs: testAssemblyVer2: | - **\bin\**\netcoreapp3.1\DurableTask.AzureStorage.Tests.dll - **\bin\**\netcoreapp3.1\DurableTask.Core.Tests.dll - **\bin\**\netcoreapp3.1\DurableTask.Emulator.Tests.dll + ${{ parameters.testAssembly }} !**\obj\** testFiltercriteria: 'TestCategory!=DisabledInCI' vsTestVersion: 17.0 From fb93f94fd17d9c3e7519db1c4554a324c30b5d28 Mon Sep 17 00:00:00 2001 From: David Justo Date: Sat, 13 Jul 2024 01:06:53 -0700 Subject: [PATCH 56/62] remove periods --- eng/ci/public-build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index 2292aa19e..ba5db1a83 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -50,7 +50,7 @@ extends: skipBuildTagsForGitHubPullRequests: ${{ variables['System.PullRequest.IsFork'] }} stages: - - stage: DTFx.CoreValidate + - stage: DTFxCoreValidate jobs: - job: Validate strategy: @@ -68,7 +68,7 @@ extends: - template: /eng/templates/test.yml@self parameters: testAssembly: '**\bin\**\netcoreapp3.1\DurableTask.Core.Tests.dll' - - stage: DTFx.ASValidate + - stage: DTFxASValidate jobs: - job: Validate strategy: @@ -86,7 +86,7 @@ extends: - template: /eng/templates/test.yml@self parameters: testAssembly: '**\bin\**\netcoreapp3.1\DurableTask.AzureStorage.Tests.dll' - - stage: DTFx.EmulatorValidate + - stage: DTFxEmulatorValidate jobs: - job: Validate strategy: From 2039e9950112f4eb5d83a82cbbbcdd3a12e6a4ff Mon Sep 17 00:00:00 2001 From: David Justo Date: Sat, 13 Jul 2024 01:12:34 -0700 Subject: [PATCH 57/62] make stages parallel --- eng/ci/public-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index ba5db1a83..4d4c1b0bc 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -69,6 +69,7 @@ extends: parameters: testAssembly: '**\bin\**\netcoreapp3.1\DurableTask.Core.Tests.dll' - stage: DTFxASValidate + dependsOn: [] jobs: - job: Validate strategy: @@ -87,6 +88,7 @@ extends: parameters: testAssembly: '**\bin\**\netcoreapp3.1\DurableTask.AzureStorage.Tests.dll' - stage: DTFxEmulatorValidate + dependsOn: [] jobs: - job: Validate strategy: From 8c845975dadd85bc39a079419776610fdb92ebdf Mon Sep 17 00:00:00 2001 From: David Justo Date: Sat, 13 Jul 2024 08:52:37 -0700 Subject: [PATCH 58/62] add debug info --- .../LocalOrchestrationService.cs | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.Emulator/LocalOrchestrationService.cs b/src/DurableTask.Emulator/LocalOrchestrationService.cs index 50e43ecd5..bc938181e 100644 --- a/src/DurableTask.Emulator/LocalOrchestrationService.cs +++ b/src/DurableTask.Emulator/LocalOrchestrationService.cs @@ -316,7 +316,46 @@ public Task SendTaskOrchestrationMessageBatchAsync(params TaskMessage[] messages if (ret == timeOutTask) { - throw new TimeoutException("timed out or canceled while waiting for orchestration to complete"); + // for debug + // might have finished already + string debug = ""; + lock (this.thisLock) + { + if (this.instanceStore.ContainsKey(instanceId)) + { + Dictionary stateMap = this.instanceStore[instanceId]; + + if (stateMap != null && stateMap.Count > 0) + { + OrchestrationState state = null; + if (string.IsNullOrWhiteSpace(executionId)) + { + IOrderedEnumerable sortedStateMap = stateMap.Values.OrderByDescending(os => os.CreatedTime); + state = sortedStateMap.First(); + } + else + { + if (stateMap.ContainsKey(executionId)) + { + state = this.instanceStore[instanceId][executionId]; + } + } + + if (state != null + && state.OrchestrationStatus != OrchestrationStatus.Running + && state.OrchestrationStatus != OrchestrationStatus.Pending) + { + // if only master id was specified then continueAsNew is a not a terminal state + if (!(string.IsNullOrWhiteSpace(executionId) && state.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)) + { + tcs.TrySetResult(state); + } + } + debug = $"State: {state.OrchestrationStatus}, LastUpdate: {state.LastUpdatedTime}"; + } + } + } + throw new TimeoutException($"timed out or canceled while waiting for orchestration to complete. Debug {debug}"); } cts.Cancel(); From 9e8f5c5069ac041c5f052aeb12dfff2cea0dc754 Mon Sep 17 00:00:00 2001 From: David Justo Date: Sat, 13 Jul 2024 08:53:07 -0700 Subject: [PATCH 59/62] undo --- .../LocalOrchestrationService.cs | 41 +------------------ 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/src/DurableTask.Emulator/LocalOrchestrationService.cs b/src/DurableTask.Emulator/LocalOrchestrationService.cs index bc938181e..50e43ecd5 100644 --- a/src/DurableTask.Emulator/LocalOrchestrationService.cs +++ b/src/DurableTask.Emulator/LocalOrchestrationService.cs @@ -316,46 +316,7 @@ public Task SendTaskOrchestrationMessageBatchAsync(params TaskMessage[] messages if (ret == timeOutTask) { - // for debug - // might have finished already - string debug = ""; - lock (this.thisLock) - { - if (this.instanceStore.ContainsKey(instanceId)) - { - Dictionary stateMap = this.instanceStore[instanceId]; - - if (stateMap != null && stateMap.Count > 0) - { - OrchestrationState state = null; - if (string.IsNullOrWhiteSpace(executionId)) - { - IOrderedEnumerable sortedStateMap = stateMap.Values.OrderByDescending(os => os.CreatedTime); - state = sortedStateMap.First(); - } - else - { - if (stateMap.ContainsKey(executionId)) - { - state = this.instanceStore[instanceId][executionId]; - } - } - - if (state != null - && state.OrchestrationStatus != OrchestrationStatus.Running - && state.OrchestrationStatus != OrchestrationStatus.Pending) - { - // if only master id was specified then continueAsNew is a not a terminal state - if (!(string.IsNullOrWhiteSpace(executionId) && state.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)) - { - tcs.TrySetResult(state); - } - } - debug = $"State: {state.OrchestrationStatus}, LastUpdate: {state.LastUpdatedTime}"; - } - } - } - throw new TimeoutException($"timed out or canceled while waiting for orchestration to complete. Debug {debug}"); + throw new TimeoutException("timed out or canceled while waiting for orchestration to complete"); } cts.Cancel(); From 33764d0980cd1fd9fc5c79027d1f0a2f09443dfb Mon Sep 17 00:00:00 2001 From: Will Sugarman Date: Wed, 17 Jul 2024 17:37:01 -0700 Subject: [PATCH 60/62] Fix DurableTaskStorageException Constructor Null Handling (#1140) --- .../DurableTaskStorageExceptionTests.cs | 44 +++++++++++++++++++ .../DurableTask.AzureStorage.csproj | 2 +- .../Storage/DurableTaskStorageException.cs | 14 +++--- 3 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 Test/DurableTask.AzureStorage.Tests/Storage/DurableTaskStorageExceptionTests.cs diff --git a/Test/DurableTask.AzureStorage.Tests/Storage/DurableTaskStorageExceptionTests.cs b/Test/DurableTask.AzureStorage.Tests/Storage/DurableTaskStorageExceptionTests.cs new file mode 100644 index 000000000..f57918638 --- /dev/null +++ b/Test/DurableTask.AzureStorage.Tests/Storage/DurableTaskStorageExceptionTests.cs @@ -0,0 +1,44 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.AzureStorage.Tests.Storage +{ + using System.Net; + using Azure; + using Azure.Storage.Blobs.Models; + using DurableTask.AzureStorage.Storage; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class DurableTaskStorageExceptionTests + { + [TestMethod] + public void MissingRequestFailedException() + { + DurableTaskStorageException exception = new((RequestFailedException)null!); + Assert.AreEqual(0, exception.HttpStatusCode); + Assert.IsFalse(exception.LeaseLost); + } + + [DataTestMethod] + [DataRow(true, HttpStatusCode.Conflict, nameof(BlobErrorCode.LeaseLost))] + [DataRow(false, HttpStatusCode.Conflict, nameof(BlobErrorCode.LeaseNotPresentWithBlobOperation))] + [DataRow(false, HttpStatusCode.NotFound, nameof(BlobErrorCode.BlobNotFound))] + public void ValidRequestFailedException(bool expectedLease, HttpStatusCode statusCode, string errorCode) + { + DurableTaskStorageException exception = new(new RequestFailedException((int)statusCode, "Error!", errorCode, innerException: null)); + Assert.AreEqual((int)statusCode, exception.HttpStatusCode); + Assert.AreEqual(expectedLease, exception.LeaseLost); + } + } +} diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 19d08a87e..c64802cd1 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -40,7 +40,7 @@ - + diff --git a/src/DurableTask.AzureStorage/Storage/DurableTaskStorageException.cs b/src/DurableTask.AzureStorage/Storage/DurableTaskStorageException.cs index 6c31e2287..af5e4d289 100644 --- a/src/DurableTask.AzureStorage/Storage/DurableTaskStorageException.cs +++ b/src/DurableTask.AzureStorage/Storage/DurableTaskStorageException.cs @@ -24,23 +24,23 @@ public DurableTaskStorageException() { } - public DurableTaskStorageException(string message) + public DurableTaskStorageException(string? message) : base(message) { } - public DurableTaskStorageException(string message, Exception inner) + public DurableTaskStorageException(string? message, Exception? inner) : base(message, inner) { } - public DurableTaskStorageException(RequestFailedException requestFailedException) - : base(requestFailedException.Message, requestFailedException) + public DurableTaskStorageException(RequestFailedException? requestFailedException) + : base("An error occurred while communicating with Azure Storage", requestFailedException) { - this.HttpStatusCode = requestFailedException.Status; - if (requestFailedException?.ErrorCode == BlobErrorCode.LeaseLost) + if (requestFailedException != null) { - LeaseLost = true; + this.HttpStatusCode = requestFailedException.Status; + this.LeaseLost = requestFailedException.ErrorCode != null && requestFailedException.ErrorCode == BlobErrorCode.LeaseLost; } } From e73e75f48fbda8106e0fc088c2f8e01312ee09e7 Mon Sep 17 00:00:00 2001 From: David Justo Date: Wed, 17 Jul 2024 21:22:21 -0700 Subject: [PATCH 61/62] Add 1ES CI for DTFx.AS v2 (#1139) --- eng/ci/public-build.yml | 8 +++--- eng/templates/test.yml | 24 ++++++++--------- .../DurableTask.AzureStorage.csproj | 7 ++++- .../Tracking/AzureTableTrackingStore.cs | 2 +- src/DurableTask.Core/DurableTask.Core.csproj | 7 ++++- .../TaskActivityDispatcher.cs | 1 + .../TaskOrchestrationDispatcher.cs | 1 + src/DurableTask.Core/Tracing/TraceHelper.cs | 14 +++++----- .../AzureStorageScenarioTests.cs | 3 +++ .../DurableTask.AzureStorage.Tests.csproj | 20 +++++++------- .../TestHelpers.cs | 6 +++-- .../TestTablePartitionManager.cs | 2 -- .../DispatcherMiddlewareTests.cs | 27 ++++++++++++++----- .../DurableTask.Core.Tests.csproj | 18 +++++-------- .../ExceptionHandlingIntegrationTests.cs | 13 +++++++-- .../RetryInterceptorTests.cs | 4 +-- test/DurableTask.Core.Tests/app.config | 6 ----- .../DurableTask.Emulator.Tests.csproj | 4 +-- .../SimpleOrchestrations.cs | 5 ++++ 19 files changed, 101 insertions(+), 71 deletions(-) diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index 4d4c1b0bc..48af28311 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -31,7 +31,7 @@ resources: ref: refs/tags/release extends: - # The template we extend injects compliance-checks into the pipleine, such as SDL and CodeQL + # The template we extend injects compliance-checks into the pipeline, such as SDL and CodeQL template: v1/1ES.Unofficial.PipelineTemplate.yml@1es parameters: pool: @@ -67,7 +67,7 @@ extends: # Run tests - template: /eng/templates/test.yml@self parameters: - testAssembly: '**\bin\**\netcoreapp3.1\DurableTask.Core.Tests.dll' + testAssembly: '**\bin\**\DurableTask.Core.Tests.dll' - stage: DTFxASValidate dependsOn: [] jobs: @@ -86,7 +86,7 @@ extends: # Run tests - template: /eng/templates/test.yml@self parameters: - testAssembly: '**\bin\**\netcoreapp3.1\DurableTask.AzureStorage.Tests.dll' + testAssembly: '**\bin\**\DurableTask.AzureStorage.Tests.dll' - stage: DTFxEmulatorValidate dependsOn: [] jobs: @@ -105,4 +105,4 @@ extends: # Run tests - template: /eng/templates/test.yml@self parameters: - testAssembly: '**\bin\**\netcoreapp3.1\DurableTask.Emulator.Tests.dll' \ No newline at end of file + testAssembly: '**\bin\**\DurableTask.Emulator.Tests.dll' \ No newline at end of file diff --git a/eng/templates/test.yml b/eng/templates/test.yml index 2cd5615f9..59de51f90 100644 --- a/eng/templates/test.yml +++ b/eng/templates/test.yml @@ -20,16 +20,14 @@ steps: - task: VSTest@2 displayName: 'Run tests' inputs: - testAssemblyVer2: | - ${{ parameters.testAssembly }} - !**\obj\** - testFiltercriteria: 'TestCategory!=DisabledInCI' - vsTestVersion: 17.0 - distributionBatchType: basedOnExecutionTime - platform: 'any cpu' - configuration: 'Debug' - diagnosticsEnabled: True - collectDumpOn: always - rerunFailedTests: true - rerunFailedThreshold: 30 - rerunMaxAttempts: 3 \ No newline at end of file + testAssemblyVer2: ${{ parameters.testAssembly }} + testFiltercriteria: 'TestCategory!=DisabledInCI' + vsTestVersion: 17.0 + distributionBatchType: basedOnExecutionTime + platform: 'any cpu' + configuration: 'Debug' + diagnosticsEnabled: True + collectDumpOn: always + rerunFailedTests: true + rerunFailedThreshold: 30 + rerunMaxAttempts: 3 \ No newline at end of file diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index c64802cd1..acb32db3b 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -13,7 +13,8 @@ true embedded false - + .\README.md + NU5125;NU5048;CS7035 @@ -53,6 +54,10 @@ + + + + true diff --git a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs index 4b7e1af7c..bcb3f1f2c 100644 --- a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs @@ -513,7 +513,7 @@ public override async IAsyncEnumerable GetStateAsync(IEnumer yield break; } - IEnumerable> instanceQueries = instanceIds.Select(instance => this.GetStateAsync(instance, allExecutions: true, fetchInput: false, cancellationToken).SingleAsync().AsTask()); + IEnumerable> instanceQueries = instanceIds.Select(instance => this.GetStateAsync(instance, allExecutions: true, fetchInput: false, cancellationToken).SingleOrDefaultAsync().AsTask()); foreach (OrchestrationState state in await Task.WhenAll(instanceQueries)) { if (state != null) diff --git a/src/DurableTask.Core/DurableTask.Core.csproj b/src/DurableTask.Core/DurableTask.Core.csproj index 786e1e42c..830f0b116 100644 --- a/src/DurableTask.Core/DurableTask.Core.csproj +++ b/src/DurableTask.Core/DurableTask.Core.csproj @@ -25,6 +25,7 @@ $(VersionPrefix).$(FileVersionRevision) $(MajorVersion).$(MinorVersion).0.0 + .\README.md @@ -38,12 +39,16 @@ - + + + + + true diff --git a/src/DurableTask.Core/TaskActivityDispatcher.cs b/src/DurableTask.Core/TaskActivityDispatcher.cs index c9e402401..1c22c307f 100644 --- a/src/DurableTask.Core/TaskActivityDispatcher.cs +++ b/src/DurableTask.Core/TaskActivityDispatcher.cs @@ -23,6 +23,7 @@ namespace DurableTask.Core using DurableTask.Core.Logging; using DurableTask.Core.Middleware; using DurableTask.Core.Tracing; + using ActivityStatusCode = Tracing.ActivityStatusCode; /// /// Dispatcher for task activities to handle processing and renewing of work items diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index fc051994f..61864a1a5 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -28,6 +28,7 @@ namespace DurableTask.Core using DurableTask.Core.Middleware; using DurableTask.Core.Serializing; using DurableTask.Core.Tracing; + using ActivityStatusCode = Tracing.ActivityStatusCode; /// /// Dispatcher for orchestrations to handle processing and renewing, completion of orchestration events diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 143ba8af6..37f09c9d5 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -85,7 +85,7 @@ public class TraceHelper DateTimeOffset startTime = startEvent.ParentTraceContext.ActivityStartTime ?? default; Activity? activity = ActivityTraceSource.StartActivity( - name: activityName, + activityName, kind: activityKind, parentContext: activityContext, startTime: startTime); @@ -139,7 +139,7 @@ public class TraceHelper } Activity? newActivity = ActivityTraceSource.StartActivity( - name: CreateSpanName(TraceActivityConstants.Activity, scheduledEvent.Name, scheduledEvent.Version), + CreateSpanName(TraceActivityConstants.Activity, scheduledEvent.Name, scheduledEvent.Version), kind: ActivityKind.Server, parentContext: activityContext); @@ -180,7 +180,7 @@ public class TraceHelper } Activity? newActivity = ActivityTraceSource.StartActivity( - name: CreateSpanName(TraceActivityConstants.Activity, taskScheduledEvent.Name, taskScheduledEvent.Version), + CreateSpanName(TraceActivityConstants.Activity, taskScheduledEvent.Name, taskScheduledEvent.Version), kind: ActivityKind.Client, startTime: taskScheduledEvent.Timestamp, parentContext: Activity.Current?.Context ?? default); @@ -274,7 +274,7 @@ public class TraceHelper } Activity? activity = ActivityTraceSource.StartActivity( - name: CreateSpanName(TraceActivityConstants.Orchestration, createdEvent.Name, createdEvent.Version), + CreateSpanName(TraceActivityConstants.Orchestration, createdEvent.Name, createdEvent.Version), kind: ActivityKind.Client, startTime: createdEvent.Timestamp, parentContext: Activity.Current?.Context ?? default); @@ -358,7 +358,7 @@ public class TraceHelper string? targetInstanceId) { Activity? newActivity = ActivityTraceSource.StartActivity( - name: CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaisedEvent.Name, null), + CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaisedEvent.Name, null), kind: ActivityKind.Producer, parentContext: Activity.Current?.Context ?? default); @@ -391,7 +391,7 @@ public class TraceHelper internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { Activity? newActivity = ActivityTraceSource.StartActivity( - name: CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaised.Name, null), + CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaised.Name, null), kind: ActivityKind.Producer, parentContext: Activity.Current?.Context ?? default, tags: new KeyValuePair[] @@ -418,7 +418,7 @@ public class TraceHelper TimerFiredEvent timerFiredEvent) { Activity? newActivity = ActivityTraceSource.StartActivity( - name: CreateTimerSpanName(orchestrationName), + CreateTimerSpanName(orchestrationName), kind: ActivityKind.Internal, startTime: startTime, parentContext: Activity.Current?.Context ?? default); diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 8e8f7ac69..7dadb01ee 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -2252,6 +2252,7 @@ public async Task ScheduledStart_Activity_GetStatus_Returns_ScheduledStart(bool /// End-to-end test which validates a simple orchestrator function that calls an activity function /// and checks the OpenTelemetry trace information /// + [TestCategory("DisabledInCI")] [DataTestMethod] [DataRow(true)] [DataRow(false)] @@ -2344,6 +2345,7 @@ public async Task OpenTelemetry_SayHelloWithActivity(bool enableExtendedSessions /// End-to-end test which validates a simple orchestrator function that waits for an external event /// raised through the RaiseEvent API and checks the OpenTelemetry trace information /// + [TestCategory("DisabledInCI")] [DataTestMethod] [DataRow(true)] [DataRow(false)] @@ -2445,6 +2447,7 @@ public async Task OpenTelemetry_ExternalEvent_RaiseEvent(bool enableExtendedSess /// End-to-end test which validates a simple orchestrator function that waits for an external event /// raised by calling SendEvent and checks the OpenTelemetry trace information /// + [TestCategory("DisabledInCI")] [DataTestMethod] [DataRow(true)] [DataRow(false)] diff --git a/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj b/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj index 0fd66ca7f..7759315b9 100644 --- a/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj +++ b/test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj @@ -5,28 +5,31 @@ netcoreapp3.1;net462 - - - - + + - - - + + + + + + + @@ -48,9 +51,6 @@ Always - - PreserveNewest - \ No newline at end of file diff --git a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs index 3608d0120..1b8740e2b 100644 --- a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs +++ b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs @@ -31,6 +31,8 @@ static class TestHelpers { string storageConnectionString = GetTestStorageAccountConnectionString(); + // TODO: update Microsoft.Extensions.Logging to avoid the following warning suppression +#pragma warning disable CS0618 // Type or member is obsolete var settings = new AzureStorageOrchestrationServiceSettings { ExtendedSessionIdleTimeout = TimeSpan.FromSeconds(extendedSessionTimeoutInSeconds), @@ -40,9 +42,9 @@ static class TestHelpers TaskHubName = GetTestTaskHubName(), // Setting up a logger factory to enable the new DurableTask.Core logs - // TODO: Add a logger provider so we can collect these logs in memory. - LoggerFactory = new LoggerFactory(), + LoggerFactory = new LoggerFactory().AddConsole(LogLevel.Trace), }; +#pragma warning restore CS0618 // Type or member is obsolete // Give the caller a chance to make test-specific changes to the settings modifySettingsAction?.Invoke(settings); diff --git a/test/DurableTask.AzureStorage.Tests/TestTablePartitionManager.cs b/test/DurableTask.AzureStorage.Tests/TestTablePartitionManager.cs index c7d945aab..8afbfc545 100644 --- a/test/DurableTask.AzureStorage.Tests/TestTablePartitionManager.cs +++ b/test/DurableTask.AzureStorage.Tests/TestTablePartitionManager.cs @@ -446,7 +446,6 @@ public async Task TestKillOneWorker() // Start with four workers and four partitions. Then kill three workers. // Test that the remaining worker will take all the partitions. - [TestCategory("DisabledInCI")] [TestMethod] public async Task TestKillThreeWorker() { @@ -586,7 +585,6 @@ public async Task TestUnhealthyWorker() /// Ensure that all instances should be processed sucessfully. /// /// - [TestCategory("DisabledInCI")] [TestMethod] public async Task EnsureOwnedQueueExclusive() { diff --git a/test/DurableTask.Core.Tests/DispatcherMiddlewareTests.cs b/test/DurableTask.Core.Tests/DispatcherMiddlewareTests.cs index ad89efc92..cee91ef9e 100644 --- a/test/DurableTask.Core.Tests/DispatcherMiddlewareTests.cs +++ b/test/DurableTask.Core.Tests/DispatcherMiddlewareTests.cs @@ -10,6 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // ---------------------------------------------------------------------------------- +#if !NET462 // for some reasons these tests are not discoverable on 1ES, leading to the test getting aborted. TODO: Needs investigation #nullable enable namespace DurableTask.Core.Tests { @@ -27,6 +28,8 @@ namespace DurableTask.Core.Tests using DurableTask.Core.History; using DurableTask.Emulator; using DurableTask.Test.Orchestrations; + using Microsoft.Extensions.Logging; + using Microsoft.Extensions.Logging.Console; using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] @@ -36,23 +39,34 @@ public class DispatcherMiddlewareTests TaskHubClient client = null!; [TestInitialize] - public async Task Initialize() + public void InitializeTests() { + // configure logging so traces are emitted during tests. + // This facilitates debugging when tests fail. + + // TODO: update Microsoft.Extensions.Logging to avoid the following warning suppression +#pragma warning disable CS0618 // Type or member is obsolete + var loggerFactory = new LoggerFactory().AddConsole(LogLevel.Trace); +#pragma warning restore CS0618 // Type or member is obsolete var service = new LocalOrchestrationService(); - this.worker = new TaskHubWorker(service); + this.worker = new TaskHubWorker(service, loggerFactory); - await this.worker + // We use `GetAwaiter().GetResult()` because otherwise this method will fail with: + // "X has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter." + this.worker .AddTaskOrchestrations(typeof(SimplestGreetingsOrchestration), typeof(ParentWorkflow), typeof(ChildWorkflow)) .AddTaskActivities(typeof(SimplestGetUserTask), typeof(SimplestSendGreetingTask)) - .StartAsync(); + .StartAsync().GetAwaiter().GetResult(); this.client = new TaskHubClient(service); } [TestCleanup] - public async Task TestCleanup() + public void CleanupTests() { - await this.worker!.StopAsync(true); + // We use `GetAwaiter().GetResult()` because otherwise this method will fail with: + // "X has wrong signature. The method must be non-static, public, does not return a value and should not take any parameter." + this.worker!.StopAsync(true).GetAwaiter().GetResult(); } [TestMethod] @@ -440,3 +454,4 @@ public async Task MockActivityOrchestration() } } } +#endif \ No newline at end of file diff --git a/test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj b/test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj index 8796c5fe8..799f6d93d 100644 --- a/test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj +++ b/test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj @@ -5,19 +5,13 @@ netcoreapp3.1;net462 - - - - - - - - + - - - - + + + + + diff --git a/test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs b/test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs index ed41ea931..f22825166 100644 --- a/test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs +++ b/test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs @@ -19,6 +19,7 @@ namespace DurableTask.Core.Tests using System.Threading.Tasks; using DurableTask.Core.Exceptions; using DurableTask.Emulator; + using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json; @@ -32,9 +33,17 @@ public class ExceptionHandlingIntegrationTests public ExceptionHandlingIntegrationTests() { + // configure logging so traces are emitted during tests. + // This facilitates debugging when tests fail. + + // TODO: update Microsoft.Extensions.Logging to avoid the following warning suppression +#pragma warning disable CS0618 // Type or member is obsolete + var loggerFactory = new LoggerFactory().AddConsole(LogLevel.Trace); +#pragma warning restore CS0618 // Type or member is obsolete + var service = new LocalOrchestrationService(); - this.worker = new TaskHubWorker(service); - this.client = new TaskHubClient(service); + this.worker = new TaskHubWorker(service, loggerFactory); + this.client = new TaskHubClient(service, loggerFactory: loggerFactory); } [DataTestMethod] diff --git a/test/DurableTask.Core.Tests/RetryInterceptorTests.cs b/test/DurableTask.Core.Tests/RetryInterceptorTests.cs index a8a711724..eddd9b12c 100644 --- a/test/DurableTask.Core.Tests/RetryInterceptorTests.cs +++ b/test/DurableTask.Core.Tests/RetryInterceptorTests.cs @@ -31,7 +31,7 @@ public async Task Invoke_WithFailingRetryCall_ShouldThrowCorrectException() await Assert.ThrowsExceptionAsync(Invoke, "Interceptor should throw the original exception after exceeding max retry attempts."); } - [TestMethod] + [DataTestMethod] [DataRow(1)] [DataRow(2)] [DataRow(3)] @@ -59,7 +59,7 @@ public async Task Invoke_WithFailingRetryCall_ShouldHaveCorrectNumberOfCalls(int Assert.AreEqual(maxAttempts, callCount, 0, $"There should be {maxAttempts} function calls for {maxAttempts} max attempts."); } - [TestMethod] + [DataTestMethod] [DataRow(1)] [DataRow(2)] [DataRow(3)] diff --git a/test/DurableTask.Core.Tests/app.config b/test/DurableTask.Core.Tests/app.config index d1d912b64..8120452b7 100644 --- a/test/DurableTask.Core.Tests/app.config +++ b/test/DurableTask.Core.Tests/app.config @@ -6,11 +6,5 @@ - - - - - - \ No newline at end of file diff --git a/test/DurableTask.Emulator.Tests/DurableTask.Emulator.Tests.csproj b/test/DurableTask.Emulator.Tests/DurableTask.Emulator.Tests.csproj index f73cb8173..c3c6edfee 100644 --- a/test/DurableTask.Emulator.Tests/DurableTask.Emulator.Tests.csproj +++ b/test/DurableTask.Emulator.Tests/DurableTask.Emulator.Tests.csproj @@ -7,8 +7,8 @@ - - + + diff --git a/test/DurableTask.Test.Orchestrations/SimpleOrchestrations.cs b/test/DurableTask.Test.Orchestrations/SimpleOrchestrations.cs index c8505f928..2cdea2d01 100644 --- a/test/DurableTask.Test.Orchestrations/SimpleOrchestrations.cs +++ b/test/DurableTask.Test.Orchestrations/SimpleOrchestrations.cs @@ -409,6 +409,11 @@ public async override Task RunTask(OrchestrationContext context, bool us responderOrchestration = Task.FromResult("Herkimer is done"); } + // before sending the event, wait a few seconds to ensure the sub-orchestrator exists + // otherwise, we risk a race condition where the event is dicarded because the instances table + // does not yet have the sub-orchestrator instance in it. + await context.CreateTimer(context.CurrentUtcDateTime.AddSeconds(10), state: null); + // send the id of this orchestration to the responder var responderInstance = new OrchestrationInstance() { InstanceId = responderId }; context.SendEvent(responderInstance, channelName, context.OrchestrationInstance.InstanceId); From 7c04b2a8d09b346ea80ca3d5a4e6cbd53f2fe0fc Mon Sep 17 00:00:00 2001 From: David Justo Date: Wed, 17 Jul 2024 21:49:32 -0700 Subject: [PATCH 62/62] remove GH actions --- .github/workflows/validate-build-2.yml | 53 ------------------------ .github/workflows/validate-build-3.yml | 53 ------------------------ .github/workflows/validate-build-4.yml | 54 ------------------------ .github/workflows/validate-build-5.yml | 54 ------------------------ .github/workflows/validate-build.yml | 57 -------------------------- 5 files changed, 271 deletions(-) delete mode 100644 .github/workflows/validate-build-2.yml delete mode 100644 .github/workflows/validate-build-3.yml delete mode 100644 .github/workflows/validate-build-4.yml delete mode 100644 .github/workflows/validate-build-5.yml delete mode 100644 .github/workflows/validate-build.yml diff --git a/.github/workflows/validate-build-2.yml b/.github/workflows/validate-build-2.yml deleted file mode 100644 index 1f0bc4a38..000000000 --- a/.github/workflows/validate-build-2.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Validate Build (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - -on: - push: - branches: - - main - paths-ignore: [ '**.md' ] - pull_request: - branches: - - main - paths-ignore: [ '**.md' ] - -env: - solution: DurableTask.sln - config: Release - -jobs: - build: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Setup .NET - uses: actions/setup-dotnet@v3 - - - name: Set up .NET Core 2.1 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '2.1.x' - - - name: Set up .NET Core 3.1 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '3.1.x' - - - name: Restore dependencies - run: dotnet restore $solution - - - name: Build - run: dotnet build $solution #--configuration $config #--no-restore -p:FileVersionRevision=$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=true - - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: '16.x' - - - name: Install Azurite - run: npm install -g azurite - - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests and AzureStorageScenarioTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Tests & ClassName!=DurableTask.AzureStorage.Tests.AzureStorageScaleTests & ClassName!=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests" \ No newline at end of file diff --git a/.github/workflows/validate-build-3.yml b/.github/workflows/validate-build-3.yml deleted file mode 100644 index 80d72c8af..000000000 --- a/.github/workflows/validate-build-3.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Validate Build (Only AzureStorageScaleTests) - -on: - push: - branches: - - main - paths-ignore: [ '**.md' ] - pull_request: - branches: - - main - paths-ignore: [ '**.md' ] - -env: - solution: DurableTask.sln - config: Release - -jobs: - build: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Setup .NET - uses: actions/setup-dotnet@v3 - - - name: Set up .NET Core 2.1 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '2.1.x' - - - name: Set up .NET Core 3.1 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '3.1.x' - - - name: Restore dependencies - run: dotnet restore $solution - - - name: Build - run: dotnet build $solution #--configuration $config #--no-restore -p:FileVersionRevision=$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=true - - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: '16.x' - - - name: Install Azurite - run: npm install -g azurite - - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Tests except AzureStorageScaleTests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "ClassName=DurableTask.AzureStorage.Tests.AzureStorageScaleTests" diff --git a/.github/workflows/validate-build-4.yml b/.github/workflows/validate-build-4.yml deleted file mode 100644 index 64ca1d9a1..000000000 --- a/.github/workflows/validate-build-4.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Validate Build (AzureStorageScenarioTests 1/2) - -on: - push: - branches: - - main - paths-ignore: [ '**.md' ] - pull_request: - branches: - - main - paths-ignore: [ '**.md' ] - -env: - solution: DurableTask.sln - config: Release - -jobs: - build: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Setup .NET - uses: actions/setup-dotnet@v3 - - - name: Set up .NET Core 2.1 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '2.1.x' - - - name: Set up .NET Core 3.1 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '3.1.x' - - - name: Restore dependencies - run: dotnet restore $solution - - - name: Build - run: dotnet build $solution #--configuration $config #--no-restore -p:FileVersionRevision=$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=true - - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: '16.x' - - - name: Install Azurite - run: npm install -g azurite - - - name: Test DTFx.AzureStorage (AzureStorageScenarioTests 1/2) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "ClassName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests & FullyQualifiedName!=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.LargeTableTextMessagePayloads_SizeViolation_BlobUrl & FullyQualifiedName!=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.LargeTableTextMessagePayloads_FetchLargeMessages & FullyQualifiedName!=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.ScheduledStart_Inline & FullyQualifiedName!=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.ScheduledStart_Activity" \ No newline at end of file diff --git a/.github/workflows/validate-build-5.yml b/.github/workflows/validate-build-5.yml deleted file mode 100644 index 65adc228f..000000000 --- a/.github/workflows/validate-build-5.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Validate Build (AzureStorageScenarioTests 2/2) - -on: - push: - branches: - - main - paths-ignore: [ '**.md' ] - pull_request: - branches: - - main - paths-ignore: [ '**.md' ] - -env: - solution: DurableTask.sln - config: Release - -jobs: - build: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Setup .NET - uses: actions/setup-dotnet@v3 - - - name: Set up .NET Core 2.1 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '2.1.x' - - - name: Set up .NET Core 3.1 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '3.1.x' - - - name: Restore dependencies - run: dotnet restore $solution - - - name: Build - run: dotnet build $solution #--configuration $config #--no-restore -p:FileVersionRevision=$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=true - - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: '16.x' - - - name: Install Azurite - run: npm install -g azurite - - - name: Test DTFx.AzureStorage (AzureStorageScenarioTests 2/2) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 | dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.LargeTableTextMessagePayloads_SizeViolation_BlobUrl | FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.LargeTableTextMessagePayloads_FetchLargeMessages | FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.ScheduledStart_Inline | FullyQualifiedName=DurableTask.AzureStorage.Tests.AzureStorageScenarioTests.ScheduledStart_Activity" \ No newline at end of file diff --git a/.github/workflows/validate-build.yml b/.github/workflows/validate-build.yml deleted file mode 100644 index d259decb9..000000000 --- a/.github/workflows/validate-build.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Validate Build (DTFx.Core and basic DTFx.AS) - -on: - push: - branches: - - main - paths-ignore: [ '**.md' ] - pull_request: - branches: - - main - paths-ignore: [ '**.md' ] - -env: - solution: DurableTask.sln - config: Release - -jobs: - build: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Setup .NET - uses: actions/setup-dotnet@v3 - - - name: Set up .NET Core 2.1 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '2.1.x' - - - name: Set up .NET Core 3.1 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '3.1.x' - - - name: Restore dependencies - run: dotnet restore $solution - - - name: Build - run: dotnet build $solution #--configuration $config #--no-restore -p:FileVersionRevision=$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=true - - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: '16.x' - - - name: Install Azurite - run: npm install -g azurite - - - name: Test DTFx.Core - run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.Core.Tests/DurableTask.Core.Tests.csproj #--configuration $config --no-build --verbosity normal - - - name: Test DTFx.AzureStorage (DurableTask.AzureStorage.Net tests) - run: azurite --skipApiVersionCheck --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/DurableTask.AzureStorage.Tests/DurableTask.AzureStorage.Tests.csproj --filter "FullyQualifiedName~DurableTask.AzureStorage.Net" #--configuration $config --no-build --verbosity normal \ No newline at end of file