Fix scaling for empty lease. (#888)

build/common.props

@@ -3,7 +3,7 @@
     <!-- Extensions can have independent versions and only increment when released -->
     <Version>3.0.0$(VersionSuffix)</Version>
     <ExtensionsVersion>5.0.0$(VersionSuffix)</ExtensionsVersion> <!-- WebJobs.Extensions -->
-    <CosmosDBVersion>4.6.0$(VersionSuffix)</CosmosDBVersion>
+    <CosmosDBVersion>4.6.1$(VersionSuffix)</CosmosDBVersion>
     <HttpVersion>3.2.0$(VersionSuffix)</HttpVersion>
     <MobileAppsVersion>3.0.0$(VersionSuffix)</MobileAppsVersion>
     <SendGridVersion>3.0.3$(VersionSuffix)</SendGridVersion>

src/WebJobs.Extensions.CosmosDB/Trigger/CosmosDBMetricsProvider.cs

@@ -17,6 +17,8 @@ internal class CosmosDBMetricsProvider
         private readonly Container _monitoredContainer;
         private readonly Container _leaseContainer;
         private readonly string _processorName;
+        private readonly int _maxAssignWorkerOnNotFoundCount = 5;
+        private int _assignWorkerOnNotFoundCount = 0;
 
         private static readonly Dictionary<string, string> KnownDocumentClientErrors = new Dictionary<string, string>()
         {
@@ -59,7 +61,16 @@ public async Task<CosmosDBTriggerMetrics> GetMetricsAsync()
                 }
 
                 partitionCount = partitionWorkList.Count;
-                remainingWork = partitionWorkList.Sum(item => item.EstimatedLag);
+                if (partitionCount == 0)
+                {
+                    partitionCount = 1;
+                    remainingWork = 1;
+                    _logger.LogWarning(Events.OnScaling, "PartitionCount is 0, the lease container exists but it has not been initialized, scale out to 1 and wait for the first execution.");
+                }
+                else
+                {
+                    remainingWork = partitionWorkList.Sum(item => item.EstimatedLag);
+                }
             }
             catch (CosmosException cosmosException) when (cosmosException.StatusCode == HttpStatusCode.Gone)
             {
@@ -69,6 +80,19 @@ public async Task<CosmosDBTriggerMetrics> GetMetricsAsync()
                 partitionCount = 1;
                 remainingWork = 1;
             }
+            catch (CosmosException cosmosException) when (cosmosException.StatusCode == HttpStatusCode.NotFound 
+                && _assignWorkerOnNotFoundCount < _maxAssignWorkerOnNotFoundCount)
+            {
+                // An exception "Not found" may indicate that the lease container does not exist.
+                // We vote to scale out, assign a worker to create the lease container.
+                // However, it could also signal an issue with the monitoring container configuration.
+                // As a result, we make a limited number of attempts to create the lease container.
+                _assignWorkerOnNotFoundCount++;
+                _logger.LogWarning(Events.OnScaling, $"Possible non-exiting lease container detected. Trying to create the lease container, attempt '{_assignWorkerOnNotFoundCount}'",
+                    cosmosException.GetType().ToString(), cosmosException.Message);
+                partitionCount = 1;
+                remainingWork = 1;
+            }
             catch (Exception e) when (e is CosmosException || e is InvalidOperationException)
             {
                 if (!TryHandleCosmosException(e))

test/WebJobs.Extensions.CosmosDB.Tests/Trigger/CosmosDBMetricsProviderTests.cs

@@ -78,8 +78,8 @@ public async Task GetMetrics_ReturnsExpectedResult()
 
             var metrics = await _cosmosDbMetricsProvider.GetMetricsAsync();
 
-            Assert.Equal(0, metrics.PartitionCount);
-            Assert.Equal(0, metrics.RemainingWork);
+            Assert.Equal(1, metrics.PartitionCount);
+            Assert.Equal(1, metrics.RemainingWork);
             Assert.NotEqual(default(DateTime), metrics.Timestamp);
 
             _estimatorIterator
@@ -140,12 +140,12 @@ public async Task GetMetrics_HandlesExceptions()
 
             var metrics = (CosmosDBTriggerMetrics)await _cosmosDbMetricsProvider.GetMetricsAsync();
 
-            Assert.Equal(0, metrics.PartitionCount);
-            Assert.Equal(0, metrics.RemainingWork);
+            Assert.Equal(1, metrics.PartitionCount);
+            Assert.Equal(1, metrics.RemainingWork);
             Assert.NotEqual(default(DateTime), metrics.Timestamp);
 
             var warning = _loggerProvider.GetAllLogMessages().Single(p => p.Level == Microsoft.Extensions.Logging.LogLevel.Warning);
-            Assert.Equal("Please check that the CosmosDB container and leases container exist and are listed correctly in Functions config files.", warning.FormattedMessage);
+            Assert.StartsWith("Possible non-exiting lease container detected. Trying to create the lease container, attempt", warning.FormattedMessage);
             _loggerProvider.ClearAllLogMessages();
 
             await Assert.ThrowsAsync<InvalidOperationException>(async () => await _cosmosDbMetricsProvider.GetMetricsAsync());