Functions SyncTriggers improvements

src/WebJobs.Script.WebHost/Controllers/FunctionsController.cs

@@ -48,7 +48,7 @@ public FunctionsController(IWebFunctionsManager functionsManager, IWebJobsRouter
         [Authorize(Policy = PolicyNames.AdminAuthLevel)]
         public async Task<IActionResult> List(bool includeProxies = false)
         {
-            var result = await _functionsManager.GetFunctionsMetadata(Request, includeProxies);
+            var result = await _functionsManager.GetFunctionsMetadata(includeProxies);
             return Ok(result);
         }
 
@@ -132,7 +132,7 @@ public async Task<IActionResult> GetFunctionStatus(string name, [FromServices] I
             {
                 // if we don't have any errors registered, make sure the function exists
                 // before returning empty errors
-                var result = await _functionsManager.GetFunctionsMetadata(Request, includeProxies: true);
+                var result = await _functionsManager.GetFunctionsMetadata(includeProxies: true);
                 var function = result.FirstOrDefault(p => p.Name.ToLowerInvariant() == name.ToLowerInvariant());
                 if (function == null)
                 {

src/WebJobs.Script.WebHost/Controllers/KeysController.cs

@@ -10,6 +10,7 @@
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Azure.WebJobs.Script.WebHost.Management;
 using Microsoft.Azure.WebJobs.Script.WebHost.Models;
 using Microsoft.Azure.WebJobs.Script.WebHost.Properties;
 using Microsoft.Azure.WebJobs.Script.WebHost.Security.Authorization.Policies;
@@ -28,13 +29,15 @@ public class KeysController : Controller
         private readonly ILogger _logger;
         private readonly IOptions<ScriptApplicationHostOptions> _applicationOptions;
         private readonly IFileSystem _fileSystem;
+        private readonly IFunctionsSyncManager _functionsSyncManager;
 
-        public KeysController(IOptions<ScriptApplicationHostOptions> applicationOptions, ISecretManagerProvider secretManagerProvider, ILoggerFactory loggerFactory, IFileSystem fileSystem)
+        public KeysController(IOptions<ScriptApplicationHostOptions> applicationOptions, ISecretManagerProvider secretManagerProvider, ILoggerFactory loggerFactory, IFileSystem fileSystem, IFunctionsSyncManager functionsSyncManager)
         {
             _applicationOptions = applicationOptions;
             _secretManagerProvider = secretManagerProvider;
             _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryKeysController);
             _fileSystem = fileSystem;
+            _functionsSyncManager = functionsSyncManager;
         }
 
         [HttpGet]
@@ -186,11 +189,13 @@ private async Task<IActionResult> AddOrUpdateSecretAsync(string keyName, string
                 case OperationResult.Created:
                     {
                         var keyResponse = ApiModelUtility.CreateApiModel(new { name = keyName, value = operationResult.Secret }, Request);
+                        await _functionsSyncManager.TrySyncTriggersAsync();
                         return Created(ApiModelUtility.GetBaseUri(Request), keyResponse);
                     }
                 case OperationResult.Updated:
                     {
                         var keyResponse = ApiModelUtility.CreateApiModel(new { name = keyName, value = operationResult.Secret }, Request);
+                        await _functionsSyncManager.TrySyncTriggersAsync();
                         return Ok(keyResponse);
                     }
                 case OperationResult.NotFound:
@@ -245,6 +250,8 @@ private async Task<IActionResult> DeleteFunctionSecretAsync(string keyName, stri
                 return NotFound();
             }
 
+            await _functionsSyncManager.TrySyncTriggersAsync();
+
             _logger.LogDebug(string.Format(Resources.TraceKeysApiSecretChange, keyName, keyScope ?? "host", "Deleted"));
 
             return StatusCode(StatusCodes.Status204NoContent);

src/WebJobs.Script.WebHost/Extensions/FunctionMetadataExtensions.cs

@@ -5,8 +5,6 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
-using Microsoft.AspNetCore.Http;
-using Microsoft.Azure.WebJobs.Extensions.Http;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Management.Models;
 using Microsoft.Azure.WebJobs.Script.WebHost.Management;

src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs

@@ -128,6 +128,7 @@ internal static bool IsSyncTriggersEnvironment(IScriptWebHostEnvironment webHost
         {
             // only want to do background sync triggers when NOT
             // in standby mode and not running locally
+            // ContainerReady will be false locally - it's based on a DWAS environment flag
             return !environment.IsCoreToolsEnvironment() &&
                 !webHostEnvironment.InStandbyMode && environment.IsContainerReady();
         }
@@ -208,15 +209,64 @@ internal async Task<CloudBlockBlob> GetHashBlobAsync()
             return _hashBlob;
         }
 
-        public async Task<JArray> GetSyncTriggersPayload()
+        public async Task<JObject> GetSyncTriggersPayload()
         {
+            // don't include proxies in cache data - portal
             var hostOptions = _applicationHostOptions.CurrentValue.ToHostOptions();
-            var functionsMetadata = WebFunctionsManager.GetFunctionsMetadata(hostOptions, _workerConfigs, _logger, includeProxies: true);
+            var functionsMetadata = WebFunctionsManager.GetFunctionsMetadata(hostOptions, _workerConfigs, _logger);
+            var secretsStorageType = Environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageType);
 
             // Add trigger information used by the ScaleController
             JObject result = new JObject();
             var triggers = await GetFunctionTriggers(functionsMetadata, hostOptions);
-            return new JArray(triggers);
+            result.Add("triggers", new JArray(triggers));
+
+            // Add functions details to the payload
+            JObject functions = new JObject();
+            string routePrefix = await WebFunctionsManager.GetRoutePrefix(hostOptions.RootScriptPath);
+            var functionDetails = await WebFunctionsManager.GetFunctionMetadataResponse(functionsMetadata, hostOptions);
+            result.Add("functions", new JArray(functionDetails.Select(p => JObject.FromObject(p))));
+
+            // Add functions secrets to the payload
+            // Only secret types we own/control can we cache directly
+            // Encryption is handled by Antares before storage
+            if (string.IsNullOrEmpty(secretsStorageType) ||
+                string.Compare(secretsStorageType, "files", StringComparison.OrdinalIgnoreCase) == 0 ||
+                string.Compare(secretsStorageType, "blob", StringComparison.OrdinalIgnoreCase) == 0)
+            {
+                JObject secrets = new JObject();
+                result.Add("secrets", secrets);
+
+                // add host secrets
+                var hostSecretsInfo = await _secretManagerProvider.Current.GetHostSecretsAsync();
+                var hostSecrets = new JObject();
+                hostSecrets.Add("master", hostSecretsInfo.MasterKey);
+                hostSecrets.Add("function", JObject.FromObject(hostSecretsInfo.FunctionKeys));
+                hostSecrets.Add("system", JObject.FromObject(hostSecretsInfo.SystemKeys));
+                secrets.Add("host", hostSecrets);
+
+                // add function secrets
+                var functionSecrets = new JArray();
+                var httpFunctions = functionsMetadata.Where(p => !p.IsProxy && p.InputBindings.Any(q => q.IsTrigger && string.Compare(q.Type, "httptrigger", StringComparison.OrdinalIgnoreCase) == 0)).Select(p => p.Name);
+                foreach (var functionName in httpFunctions)
+                {
+                    var currSecrets = await _secretManagerProvider.Current.GetFunctionSecretsAsync(functionName);
+                    var currElement = new JObject()
+                    {
+                        { "name", functionName },
+                        { "secrets", JObject.FromObject(currSecrets) }
+                    };
+                    functionSecrets.Add(currElement);
+                }
+                secrets.Add("function", functionSecrets);
+            }
+            else
+            {
+                // TODO: handle other external key storage types
+                // like KeyVault when the feature comes online
+            }
+
+            return result;
         }
 
         internal async Task<IEnumerable<JObject>> GetFunctionTriggers(IEnumerable<FunctionMetadata> functionsMetadata, ScriptJobHostOptions hostOptions)
@@ -328,7 +378,11 @@ private async Task<(bool, string)> SetTriggersAsync(string content)
         {
             var token = SimpleWebTokenHelper.CreateToken(DateTime.UtcNow.AddMinutes(5));
 
-            _logger.LogDebug($"SyncTriggers content: {content}");
+            // sanitize the content before logging
+            var sanitizedContent = JObject.Parse(content);
+            sanitizedContent.Remove("secrets");
+            string sanitizedContentString = sanitizedContent.ToString();
+            _logger.LogDebug($"SyncTriggers content: {sanitizedContentString}");
 
             using (var request = BuildSetTriggersRequest())
             {
@@ -357,4 +411,4 @@ public void Dispose()
             _syncSemaphore.Dispose();
         }
     }
-}
+}

src/WebJobs.Script.WebHost/Management/IWebFunctionsManager.cs

@@ -10,7 +10,7 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost.Management
 {
     public interface IWebFunctionsManager
     {
-        Task<IEnumerable<FunctionMetadataResponse>> GetFunctionsMetadata(HttpRequest request, bool includeProxies);
+        Task<IEnumerable<FunctionMetadataResponse>> GetFunctionsMetadata(bool includeProxies);
 
         Task<(bool, FunctionMetadataResponse)> TryGetFunction(string name, HttpRequest request);
 

src/WebJobs.Script.WebHost/Management/WebFunctionsManager.cs

@@ -38,31 +38,21 @@ public WebFunctionsManager(IOptionsMonitor<ScriptApplicationHostOptions> applica
             _functionsSyncManager = functionsSyncManager;
         }
 
-        /// <summary>
-        /// Calls into ScriptHost to retrieve list of FunctionMetadata
-        /// and maps them to FunctionMetadataResponse.
-        /// </summary>
-        /// <param name="request">Current HttpRequest for figuring out baseUrl</param>
-        /// <returns>collection of FunctionMetadataResponse</returns>
-        public async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionsMetadata(HttpRequest request, bool includeProxies)
-        {
-            var baseUrl = $"{request.Scheme}://{request.Host}";
-            return await GetFunctionsMetadata(baseUrl, includeProxies);
-        }
-
-        public async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionsMetadata(string baseUrl, bool includeProxies)
+        public async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionsMetadata(bool includeProxies)
         {
             var hostOptions = _applicationHostOptions.CurrentValue.ToHostOptions();
+            var functionsMetadata = GetFunctionsMetadata(hostOptions, _workerConfigs, _logger, includeProxies);
 
-            string routePrefix = await GetRoutePrefix(hostOptions.RootScriptPath);
-            var tasks = GetFunctionsMetadata(includeProxies).Select(p => p.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl));
-            return await tasks.WhenAll();
+            return await GetFunctionMetadataResponse(functionsMetadata, hostOptions);
         }
 
-        internal IEnumerable<FunctionMetadata> GetFunctionsMetadata(bool includeProxies = false)
+        internal static async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionMetadataResponse(IEnumerable<FunctionMetadata> functionsMetadata, ScriptJobHostOptions hostOptions)
         {
-            var hostOptions = _applicationHostOptions.CurrentValue.ToHostOptions();
-            return GetFunctionsMetadata(hostOptions, _workerConfigs, _logger, includeProxies);
+            string baseUrl = GetBaseUrl();
+            string routePrefix = await GetRoutePrefix(hostOptions.RootScriptPath);
+            var tasks = functionsMetadata.Select(p => p.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl));
+
+            return await tasks.WhenAll();
         }
 
         internal static IEnumerable<FunctionMetadata> GetFunctionsMetadata(ScriptJobHostOptions hostOptions, IEnumerable<WorkerConfig> workerConfigs, ILogger logger, bool includeProxies = false)
@@ -253,5 +243,11 @@ internal static async Task<string> GetRoutePrefix(string rootScriptPath)
 
             return routePrefix;
         }
+
+        internal static string GetBaseUrl()
+        {
+            string hostName = Environment.GetEnvironmentVariable("WEBSITE_HOSTNAME") ?? "localhost";
+            return $"https://{hostName}";
+        }
     }
 }

test/WebJobs.Script.Tests/Controllers/Admin/KeysControllerTests.cs

@@ -15,6 +15,7 @@
 using Microsoft.Azure.WebJobs.Script.Eventing;
 using Microsoft.Azure.WebJobs.Script.WebHost;
 using Microsoft.Azure.WebJobs.Script.WebHost.Controllers;
+using Microsoft.Azure.WebJobs.Script.WebHost.Management;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Moq;
@@ -32,6 +33,7 @@ public class KeysControllerTests : IDisposable
         private Dictionary<string, Collection<string>> _testFunctionErrors;
         private KeysController _testController;
         private Mock<ISecretManager> _secretsManagerMock;
+        private Mock<IFunctionsSyncManager> _functionsSyncManagerMock;
 
         public KeysControllerTests()
         {
@@ -60,7 +62,9 @@ public KeysControllerTests()
             fileBase.Setup(f => f.ReadAllText(Path.Combine(rootScriptPath, "TestFunction2", ScriptConstants.FunctionMetadataFileName))).Returns("{}");
             fileBase.Setup(f => f.ReadAllText(Path.Combine(rootScriptPath, "DNE", ScriptConstants.FunctionMetadataFileName))).Throws(new DirectoryNotFoundException());
 
-            _testController = new KeysController(new OptionsWrapper<ScriptApplicationHostOptions>(settings), new TestSecretManagerProvider(_secretsManagerMock.Object), new LoggerFactory(), fileSystem.Object);
+            _functionsSyncManagerMock = new Mock<IFunctionsSyncManager>(MockBehavior.Strict);
+            _functionsSyncManagerMock.Setup(p => p.TrySyncTriggersAsync(false)).ReturnsAsync(new SyncTriggersResult { Success = true });
+            _testController = new KeysController(new OptionsWrapper<ScriptApplicationHostOptions>(settings), new TestSecretManagerProvider(_secretsManagerMock.Object), new LoggerFactory(), fileSystem.Object, _functionsSyncManagerMock.Object);
 
             var keys = new Dictionary<string, string>
             {
@@ -113,6 +117,8 @@ public async Task PutKey_NotAFunction_ReturnsNotFound()
 
             var result = (StatusCodeResult)(await _testController.Put("DNE", key.Name, key));
             Assert.Equal(StatusCodes.Status404NotFound, result.StatusCode);
+
+            _functionsSyncManagerMock.Verify(p => p.TrySyncTriggersAsync(false), Times.Never);
         }
 
         [Fact]
@@ -126,6 +132,8 @@ public async Task PutKey_Succeeds()
             var content = (JObject)result.Value;
             Assert.Equal("key2", content["name"]);
             Assert.Equal("secret2", content["value"]);
+
+            _functionsSyncManagerMock.Verify(p => p.TrySyncTriggersAsync(false), Times.Once);
         }
 
         [Fact]
@@ -135,13 +143,17 @@ public async Task DeleteKey_Succeeds()
 
             var result = (StatusCodeResult)(await _testController.Delete("TestFunction1", "key2"));
             Assert.Equal(StatusCodes.Status204NoContent, result.StatusCode);
+
+            _functionsSyncManagerMock.Verify(p => p.TrySyncTriggersAsync(false), Times.Once);
         }
 
         [Fact]
         public async Task DeleteKey_NotAFunction_ReturnsNotFound()
         {
             var result = (StatusCodeResult)(await _testController.Delete("DNE", "key2"));
             Assert.Equal(StatusCodes.Status404NotFound, result.StatusCode);
+
+            _functionsSyncManagerMock.Verify(p => p.TrySyncTriggersAsync(false), Times.Never);
         }
 
         [Fact]
@@ -151,13 +163,17 @@ public async Task DeleteKey_NotAKey_ReturnsNotFound()
 
             var result = (StatusCodeResult)(await _testController.Delete("TestFunction1", "dne"));
             Assert.Equal(StatusCodes.Status404NotFound, result.StatusCode);
+
+            _functionsSyncManagerMock.Verify(p => p.TrySyncTriggersAsync(false), Times.Never);
         }
 
         [Fact]
         public async Task DeleteKey_InvalidKeyName_ReturnsBadRequest()
         {
             var result = (BadRequestObjectResult)(await _testController.Delete("TestFunction1", "_test"));
             Assert.Equal("Invalid key name.", result.Value);
+
+            _functionsSyncManagerMock.Verify(p => p.TrySyncTriggersAsync(false), Times.Never);
         }
 
         protected virtual void Dispose(bool disposing)