Moves updating Graph DB to the GraphSelectGuidancePlugin. Closes #481 (#482)

Co-authored-by: Garry Trinder <garry@trinder365.co.uk>

Author: waldekmastykarz

dev-proxy-abstractions/MSGraphDbUtils.cs

@@ -0,0 +1,206 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using Microsoft.Data.Sqlite;
+using Microsoft.OpenApi.Models;
+using Microsoft.OpenApi.Readers;
+
+namespace Microsoft.DevProxy.Abstractions;
+
+public static class MSGraphDbUtils
+{
+    private static readonly Dictionary<string, OpenApiDocument> _openApiDocuments = new();
+    private static readonly string[] graphVersions = ["v1.0", "beta"];
+
+    private static string GetGraphOpenApiYamlFileName(string version) => $"graph-{version.Replace(".", "_")}-openapi.yaml";
+
+    // v1 refers to v1 of the db schema, not the graph version
+    public static string MSGraphDbFilePath => Path.Combine(ProxyUtils.AppFolder!, "msgraph-openapi-v1.db");
+    private static SqliteConnection? _msGraphDbConnection;
+    public static SqliteConnection MSGraphDbConnection
+    {
+        get
+        {
+            if (_msGraphDbConnection is null)
+            {
+                _msGraphDbConnection = new SqliteConnection($"Data Source={MSGraphDbFilePath}");
+                _msGraphDbConnection.Open();
+            }
+
+            return _msGraphDbConnection;
+        }
+    }
+
+    public static async Task<int> GenerateMSGraphDb(ILogger logger, bool skipIfUpdatedToday = false)
+    {
+        var appFolder = ProxyUtils.AppFolder;
+        if (string.IsNullOrEmpty(appFolder))
+        {
+            logger.LogError("App folder not found");
+            return 1;
+        }
+
+        try
+        {
+            var dbFileInfo = new FileInfo(MSGraphDbFilePath);
+            var modifiedToday = dbFileInfo.Exists && dbFileInfo.LastWriteTime.Date == DateTime.Now.Date;
+            if (modifiedToday && skipIfUpdatedToday)
+            {
+                logger.LogInfo("Microsoft Graph database already updated today");
+                return 1;
+            }
+
+            await UpdateOpenAPIGraphFilesIfNecessary(appFolder, logger);
+            await LoadOpenAPIFiles(appFolder, logger);
+            if (_openApiDocuments.Count < 1)
+            {
+                logger.LogDebug("No OpenAPI files found or couldn't load them");
+                return 1;
+            }
+
+            var dbConnection = MSGraphDbConnection;
+            CreateDb(dbConnection, logger);
+            FillData(dbConnection, logger);
+
+            logger.LogInfo("Microsoft Graph database successfully updated");
+
+            return 0;
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex.Message);
+            return 1;
+        }
+
+    }
+
+    private static void CreateDb(SqliteConnection dbConnection, ILogger logger)
+    {
+        logger.LogInfo("Creating database...");
+
+        logger.LogDebug("Dropping endpoints table...");
+        var dropTable = dbConnection.CreateCommand();
+        dropTable.CommandText = "DROP TABLE IF EXISTS endpoints";
+        dropTable.ExecuteNonQuery();
+
+        logger.LogDebug("Creating endpoints table...");
+        var createTable = dbConnection.CreateCommand();
+        // when you change the schema, increase the db version number in ProxyUtils
+        createTable.CommandText = "CREATE TABLE IF NOT EXISTS endpoints (path TEXT, graphVersion TEXT, hasSelect BOOLEAN)";
+        createTable.ExecuteNonQuery();
+
+        logger.LogDebug("Creating index on endpoints and version...");
+        // Add an index on the path and graphVersion columns
+        var createIndex = dbConnection.CreateCommand();
+        createIndex.CommandText = "CREATE INDEX IF NOT EXISTS idx_endpoints_path_version ON endpoints (path, graphVersion)";
+        createIndex.ExecuteNonQuery();
+    }
+
+    private static void FillData(SqliteConnection dbConnection, ILogger logger)
+    {
+        logger.LogInfo("Filling database...");
+
+        var i = 0;
+
+        foreach (var openApiDocument in _openApiDocuments)
+        {
+            var graphVersion = openApiDocument.Key;
+            var document = openApiDocument.Value;
+
+            logger.LogDebug($"Filling database for {graphVersion}...");
+
+            var insertEndpoint = dbConnection.CreateCommand();
+            insertEndpoint.CommandText = "INSERT INTO endpoints (path, graphVersion, hasSelect) VALUES (@path, @graphVersion, @hasSelect)";
+            insertEndpoint.Parameters.Add(new SqliteParameter("@path", null));
+            insertEndpoint.Parameters.Add(new SqliteParameter("@graphVersion", null));
+            insertEndpoint.Parameters.Add(new SqliteParameter("@hasSelect", null));
+
+            foreach (var path in document.Paths)
+            {
+                logger.LogDebug($"Endpoint {graphVersion}{path.Key}...");
+
+                // Get the GET operation for this path
+                var getOperation = path.Value.Operations.FirstOrDefault(o => o.Key == OperationType.Get).Value;
+                if (getOperation == null)
+                {
+                    logger.LogDebug($"No GET operation found for {graphVersion}{path.Key}");
+                    continue;
+                }
+
+                // Check if the GET operation has a $select parameter
+                var hasSelect = getOperation.Parameters.Any(p => p.Name == "$select");
+
+                logger.LogDebug($"Inserting endpoint {graphVersion}{path.Key} with hasSelect={hasSelect}...");
+                insertEndpoint.Parameters["@path"].Value = path.Key;
+                insertEndpoint.Parameters["@graphVersion"].Value = graphVersion;
+                insertEndpoint.Parameters["@hasSelect"].Value = hasSelect;
+                insertEndpoint.ExecuteNonQuery();
+                i++;
+            }
+        }
+
+        logger.LogInfo($"Inserted {i} endpoints in the database");
+    }
+
+    private static async Task UpdateOpenAPIGraphFilesIfNecessary(string folder, ILogger logger)
+    {
+        logger.LogInfo("Checking for updated OpenAPI files...");
+
+        foreach (var version in graphVersions)
+        {
+            try
+            {
+                var file = new FileInfo(Path.Combine(folder, GetGraphOpenApiYamlFileName(version)));
+                logger.LogDebug($"Checking for updated OpenAPI file {file}...");
+                if (file.Exists && file.LastWriteTime.Date == DateTime.Now.Date)
+                {
+                    logger.LogInfo($"File {file} already updated today");
+                    continue;
+                }
+
+                var url = $"https://raw.githubusercontent.com/microsoftgraph/msgraph-metadata/master/openapi/{version}/openapi.yaml";
+                logger.LogInfo($"Downloading OpenAPI file from {url}...");
+
+                var client = new HttpClient();
+                var response = await client.GetStringAsync(url);
+                File.WriteAllText(file.FullName, response);
+
+                logger.LogDebug($"Downloaded OpenAPI file from {url} to {file}");
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex.Message);
+            }
+        }
+    }
+
+    private static async Task LoadOpenAPIFiles(string folder, ILogger logger)
+    {
+        logger.LogInfo("Loading OpenAPI files...");
+
+        foreach (var version in graphVersions)
+        {
+            var filePath = Path.Combine(folder, GetGraphOpenApiYamlFileName(version));
+            var file = new FileInfo(filePath);
+            logger.LogDebug($"Loading OpenAPI file for {filePath}...");
+
+            if (!file.Exists)
+            {
+                logger.LogDebug($"File {filePath} does not exist");
+                continue;
+            }
+
+            try
+            {
+                var openApiDocument = await new OpenApiStreamReader().ReadAsync(file.OpenRead());
+                _openApiDocuments[version] = openApiDocument.OpenApiDocument;
+
+                logger.LogDebug($"Added OpenAPI file {filePath} for {version}");
+            }
+            catch (Exception ex)
+            {
+                logger.LogError($"Error loading OpenAPI file {filePath}: {ex.Message}");
+            }
+        }
+    }
+}

dev-proxy-abstractions/ProxyUtils.cs

@@ -26,23 +26,6 @@ public static class ProxyUtils
     private static readonly Regex deprecationRegex = new Regex("^[a-z]+_v2$", RegexOptions.IgnoreCase);
     private static readonly Regex functionCallRegex = new Regex(@"^[a-z]+\(.*\)$", RegexOptions.IgnoreCase);
 
-    // v1 refers to v1 of the db schema, not the graph version
-    public static string MsGraphDbFilePath => Path.Combine(AppFolder!, "msgraph-openapi-v1.db");
-    private static SqliteConnection? _msGraphDbConnection;
-    public static SqliteConnection MsGraphDbConnection
-    {
-        get
-        {
-            if (_msGraphDbConnection is null)
-            {
-                _msGraphDbConnection = new SqliteConnection($"Data Source={MsGraphDbFilePath}");
-                _msGraphDbConnection.Open();
-            }
-
-            return _msGraphDbConnection;
-        }
-    }
-
     // doesn't end with a path separator
     public static string? AppFolder => Path.GetDirectoryName(AppContext.BaseDirectory);
 

dev-proxy-abstractions/dev-proxy-abstractions.csproj

@@ -14,6 +14,7 @@
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
+    <PackageReference Include="Microsoft.OpenApi.Readers" Version="1.6.11" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
     <PackageReference Include="Titanium.Web.Proxy" Version="3.2.0" />
   </ItemGroup>

dev-proxy-plugins/Guidance/GraphSelectGuidancePlugin.cs

@@ -19,6 +19,13 @@ public override void Register(IPluginEvents pluginEvents,
         base.Register(pluginEvents, context, urlsToWatch, configSection);
 
         pluginEvents.AfterResponse += AfterResponse;
+
+        // for background db refresh, let's use a separate logger
+        // that only logs warnings and errors
+        var _logger2 = (ILogger)context.Logger.Clone();
+        _logger2.LogLevel = LogLevel.Warn;
+        // let's not await so that it doesn't block the proxy startup
+        _ = MSGraphDbUtils.GenerateMSGraphDb(_logger2, true);
     }
 
     private Task AfterResponse(object? sender, ProxyResponseArgs e)
@@ -61,7 +68,7 @@ private bool EndpointSupportsSelect(string graphVersion, string relativeUrl)
 
         try
         {
-            var dbConnection = ProxyUtils.MsGraphDbConnection;
+            var dbConnection = MSGraphDbUtils.MSGraphDbConnection;
             // lookup information from the database
             var selectEndpoint = dbConnection.CreateCommand();
             selectEndpoint.CommandText = "SELECT hasSelect FROM endpoints WHERE path = @path AND graphVersion = @graphVersion";