-
Notifications
You must be signed in to change notification settings - Fork 263
/
AzureStorageOptions.cs
276 lines (245 loc) · 12.8 KB
/
AzureStorageOptions.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See LICENSE in the project root for license information.
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.WindowsAzure.Storage;
namespace Microsoft.Azure.WebJobs.Extensions.DurableTask
{
/// <summary>
/// Configuration options for the Azure Storage storage provider.
/// </summary>
public class AzureStorageOptions
{
// 45 alphanumeric characters gives us a buffer in our table/queue/blob container names.
private const int MaxTaskHubNameSize = 45;
private const int MinTaskHubNameSize = 3;
private const string TaskHubPadding = "Hub";
/// <summary>
/// Gets or sets the name of the Azure Storage connection information used to manage the underlying Azure Storage resources.
/// </summary>
/// <remarks>
/// The value may refer to either a connection string or the configuration section containing connection metadata.
/// The default behavior is to use the standard <c>AzureWebJobsStorage</c> connection for all storage usage.
/// </remarks>
/// <value>
/// The name of a connection-related key that exists in the app's application settings. The value may refer to
/// a connection string or the section detailing additional connection metadata.
/// </value>
public string ConnectionName { get; set; }
/// <summary>
/// Gets or sets the name of the Azure Storage connection string used to manage the underlying Azure Storage resources.
/// </summary>
/// <remarks>
/// If not specified, the default behavior is to use the standard `AzureWebJobsStorage` connection string for all storage usage.
/// </remarks>
/// <value>The name of a connection string that exists in the app's application settings.</value>
[Obsolete("Please use ConnectionName instead.")]
public string ConnectionStringName
{
get => this.ConnectionName;
set => this.ConnectionName = value;
}
/// <summary>
/// Gets or sets the number of messages to pull from the control queue at a time.
/// </summary>
/// <remarks>
/// Messages pulled from the control queue are buffered in memory until the internal
/// dispatcher is ready to process them.
/// </remarks>
/// <value>A positive integer configured by the host. The default value is <c>32</c>.</value>
public int ControlQueueBatchSize { get; set; } = 32;
/// <summary>
/// Gets or sets the partition count for the control queue.
/// </summary>
/// <remarks>
/// Increasing the number of partitions will increase the number of workers
/// that can concurrently execute orchestrator functions. However, increasing
/// the partition count can also increase the amount of load placed on the storage
/// account and on the thread pool if the number of workers is smaller than the
/// number of partitions.
/// </remarks>
/// <value>A positive integer between 1 and 16. The default value is <c>4</c>.</value>
public int PartitionCount { get; set; } = 4;
/// <summary>
/// Gets or set the number of control queue messages that can be buffered in memory
/// at a time, at which point the dispatcher will wait before dequeuing any additional
/// messages. The default is 256. The maximum value is 1000.
/// </summary>
/// <remarks>
/// Increasing this value can improve orchestration throughput by pre-fetching more
/// orchestration messages from control queues. The downside is that it increases the
/// possibility of duplicate function executions if partition leases move between app
/// instances. This most often occurs when the number of app instances changes.
/// </remarks>
/// <value>A non-negative integer between 0 and 1000. The default value is <c>256</c>.</value>
public int ControlQueueBufferThreshold { get; set; } = 256;
/// <summary>
/// Gets or sets the visibility timeout of dequeued control queue messages.
/// </summary>
/// <value>
/// A <c>TimeSpan</c> configured by the host. The default is 5 minutes.
/// </value>
public TimeSpan ControlQueueVisibilityTimeout { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Gets or sets the visibility timeout of dequeued work item queue messages.
/// </summary>
/// <value>
/// A <c>TimeSpan</c> configured by the host. The default is 5 minutes.
/// </value>
public TimeSpan WorkItemQueueVisibilityTimeout { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Gets or sets the name of the Azure Storage connection information to use for the
/// durable tracking store (History and Instances tables).
/// </summary>
/// <remarks>
/// <para>
/// If not specified, the <see cref="ConnectionName"/> connection string is used for the durable tracking store.
/// </para>
/// <para>
/// This property is primarily useful when deploying multiple apps that need to share the same
/// tracking infrastructure. For example, when deploying two versions of an app side by side, using
/// the same tracking store allows both versions to save history into the same table, which allows
/// clients to query for instance status across all versions.
/// </para>
/// </remarks>
/// <value>
/// The name of a connection-related key that exists in the app's application settings. The value may refer to
/// a connection string or the section detailing additional connection metadata.
/// </value>
public string TrackingStoreConnectionName { get; set; }
/// <summary>
/// Gets or sets the name of the Azure Storage connection string to use for the
/// durable tracking store (History and Instances tables).
/// </summary>
/// <remarks><para>
/// If not specified, the <see cref="AzureStorageOptions.ConnectionStringName"/> connection string
/// is used for the durable tracking store.
/// </para><para>
/// This property is primarily useful when deploying multiple apps that need to share the same
/// tracking infrastructure. For example, when deploying two versions of an app side by side, using
/// the same tracking store allows both versions to save history into the same table, which allows
/// clients to query for instance status across all versions.
/// </para></remarks>
/// <value>The name of a connection string that exists in the app's application settings.</value>
[Obsolete("Please use TrackingStoreConnectionName instead.")]
public string TrackingStoreConnectionStringName
{
get => this.TrackingStoreConnectionName;
set => this.TrackingStoreConnectionName = value;
}
/// <summary>
/// Gets or sets the name prefix to use for history and instance tables in Azure Storage.
/// </summary>
/// <remarks>
/// This property is only used when <see cref="TrackingStoreConnectionName"/> is specified.
/// If no prefix is specified, the default prefix value is "DurableTask".
/// </remarks>
/// <value>The prefix to use when naming the generated Azure tables.</value>
public string TrackingStoreNamePrefix { get; set; }
/// <summary>
/// Gets or sets whether the extension will automatically fetch large messages in orchestration status
/// queries. If set to false, the extension will return large messages as a blob url.
/// </summary>
/// <value>A boolean indicating whether will automatically fetch large messages .</value>
public bool FetchLargeMessagesAutomatically { get; set; } = true;
/// <summary>
/// Gets or sets the maximum queue polling interval.
/// </summary>
/// <value>Maximum interval for polling control and work-item queues.</value>
public TimeSpan MaxQueuePollingInterval { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Determines whether or not to use the old partition management strategy, or the new
/// strategy that is more resilient to split brain problems, at the potential expense
/// of scale out performance.
/// </summary>
/// <value>A boolean indicating whether we use the legacy partition strategy. Defaults to false.</value>
public bool UseLegacyPartitionManagement { get; set; } = false;
/// <summary>
/// Throws an exception if the provided hub name violates any naming conventions for the storage provider.
/// </summary>
public void ValidateHubName(string hubName)
{
try
{
NameValidator.ValidateBlobName(hubName);
NameValidator.ValidateContainerName(hubName.ToLowerInvariant());
NameValidator.ValidateTableName(hubName);
NameValidator.ValidateQueueName(hubName.ToLowerInvariant());
}
catch (ArgumentException e)
{
throw new ArgumentException(GetTaskHubErrorString(hubName), e);
}
if (hubName.Length > 50)
{
throw new ArgumentException(GetTaskHubErrorString(hubName));
}
}
private static string GetTaskHubErrorString(string hubName)
{
return $"Task hub name '{hubName}' should contain only alphanumeric characters, start with a letter, and have length between {MinTaskHubNameSize} and {MaxTaskHubNameSize}.";
}
internal bool IsSanitizedHubName(string hubName, out string sanitizedHubName)
{
// Only alphanumeric characters are valid.
var validHubNameCharacters = hubName.ToCharArray().Where(char.IsLetterOrDigit);
if (!validHubNameCharacters.Any())
{
sanitizedHubName = "DefaultTaskHub";
return false;
}
// Azure Table storage requires that the task hub does not start with
// a number. If it does, prepend "t" to the beginning.
if (char.IsNumber(validHubNameCharacters.First()))
{
validHubNameCharacters = validHubNameCharacters.ToList();
((List<char>)validHubNameCharacters).Insert(0, 't');
}
sanitizedHubName = new string(validHubNameCharacters
.Take(MaxTaskHubNameSize)
.ToArray());
if (sanitizedHubName.Length < MinTaskHubNameSize)
{
sanitizedHubName = sanitizedHubName + TaskHubPadding;
}
if (string.Equals(hubName, sanitizedHubName))
{
return true;
}
return false;
}
/// <summary>
/// Throws an exception if any of the settings of the storage provider are invalid.
/// </summary>
public void Validate(ILogger logger)
{
if (this.ControlQueueBatchSize <= 0)
{
throw new InvalidOperationException($"{nameof(this.ControlQueueBatchSize)} must be a non-negative integer.");
}
if (this.PartitionCount < 1 || this.PartitionCount > 16)
{
throw new InvalidOperationException($"{nameof(this.PartitionCount)} must be an integer value between 1 and 16.");
}
if (this.ControlQueueVisibilityTimeout < TimeSpan.FromMinutes(1) ||
this.ControlQueueVisibilityTimeout > TimeSpan.FromMinutes(60))
{
throw new InvalidOperationException($"{nameof(this.ControlQueueVisibilityTimeout)} must be between 1 and 60 minutes.");
}
if (this.MaxQueuePollingInterval <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(this.MaxQueuePollingInterval)} must be non-negative.");
}
if (this.ControlQueueBufferThreshold < 1 || this.ControlQueueBufferThreshold > 1000)
{
throw new InvalidOperationException($"{nameof(this.ControlQueueBufferThreshold)} must be between 1 and 1000.");
}
if (this.ControlQueueBatchSize > this.ControlQueueBufferThreshold)
{
logger.LogWarning($"{nameof(this.ControlQueueBatchSize)} cannot be larger than {nameof(this.ControlQueueBufferThreshold)}. Please adjust these values in your `host.json` settings for predictable performance");
}
}
}
}