Feature 20251204 google search fix

AiCoreApi/Common/ConnectionManager.cs

@@ -6,17 +6,17 @@ namespace AiCoreApi.Common
     public class ConnectionManager: IConnectionManager
     {
         private readonly IConnectionProcessor _connectionProcessor;
-        private readonly IParametersHelper _parametersHelper;
+        private readonly IIngestionParametersHelper _ingestionParametersHelper;
 
         public ConnectionManager(
-            IConnectionProcessor connectionProcessor, 
-            IParametersHelper parametersHelper)
+            IConnectionProcessor connectionProcessor,
+            IIngestionParametersHelper ingestionParametersHelper)
         {
             _connectionProcessor = connectionProcessor;
-            _parametersHelper = parametersHelper;
+            _ingestionParametersHelper = ingestionParametersHelper;
         }
 
-        private volatile List<ConnectionModel>? _cache;
+        private List<ConnectionModel>? _cache;
         private readonly object _cacheLock = new();
 
         public async Task<List<ConnectionModel>> GetConnectionModels()
@@ -54,7 +54,7 @@ public async Task<List<ConnectionModel>> GetConnectionModels()
                 WorkspaceId = connection.WorkspaceId,
                 Content = new Dictionary<string, string>(connection.Content),
             };
-            return await _parametersHelper.ApplySecrets(connectionCopy);
+            return await _ingestionParametersHelper.ApplySecrets(connectionCopy);
         }
 
         public async Task<ConnectionModel?> GetConnection(int? workspaceId = null, int? connectionId = null, ConnectionType? connectionType = null, bool isLlmConnection = false, bool isEmbeddingConnection = false, string? connectionName = null)
@@ -67,8 +67,7 @@ public async Task<List<ConnectionModel>> GetConnectionModels()
                      && (!isLlmConnection || c.Type.IsLlmConnection())
                      && (!isEmbeddingConnection || c.Type.IsEmbeddingConnection())
                      && (string.IsNullOrEmpty(connectionName) || c.Name == connectionName)
-                 )
-                 ?? null;
+                 );
             return connection;
         }
 

AiCoreApi/Common/ParametersHelper.cs

@@ -6,41 +6,36 @@
 
 namespace AiCoreApi.Common
 {
-    public class ParametersHelper : ParametersHelperBase, IParametersHelper
+    public class IngestionParametersHelper : ParametersHelperBase, IIngestionParametersHelper
     {
-        public ParametersHelper(
+        public IngestionParametersHelper(
             RequestAccessor requestAccessor,
             ICacheAccessor cacheAccessor,
             ResponseAccessor responseAccessor,
-            ILogger<ParametersHelper> logger,
+            ILogger<IngestionParametersHelper> logger,
             IEntraTokenProvider entraTokenProvider)
             : base(requestAccessor, cacheAccessor, responseAccessor, logger, entraTokenProvider)
         {
         }
-
-        protected override string ResolveUnknownKey(string key)
-        {
-            return $"{{{{{key}}}}}";
-        }
     }
 
-    public interface IParametersHelper
+    public interface IIngestionParametersHelper
     {
         Task<ConnectionModel> ApplySecrets(ConnectionModel connectionModel);
         Task<string> ApplySecret(string value);
         Task<string> ApplyParametersAsync(string text, string debugSource, Dictionary<string, string>? parameters, Dictionary<string, string>? additionalParameters = null);
     }
 
-    public class IngestionParametersHelper : ParametersHelperBase, IIngestionParametersHelper
+    public class ParametersHelper : ParametersHelperBase, IParametersHelper
     {
         private readonly IIngestionProcessor _ingestionProcessor;
         private readonly IDataIngestionWorkerFactory _workerFactory;
 
-        public IngestionParametersHelper(
+        public ParametersHelper(
             RequestAccessor requestAccessor,
             ICacheAccessor cacheAccessor,
             ResponseAccessor responseAccessor,
-            ILogger<IngestionParametersHelper> logger,
+            ILogger<ParametersHelper> logger,
             IEntraTokenProvider entraTokenProvider,
             IIngestionProcessor ingestionProcessor,
             IDataIngestionWorkerFactory workerFactory)
@@ -89,7 +84,7 @@ private async Task<string> ResolveDataSourceValueAsync(string key)
         }
     }
 
-    public interface IIngestionParametersHelper
+    public interface IParametersHelper
     {
         Task<ConnectionModel> ApplySecrets(ConnectionModel connectionModel);
         Task<string> ApplySecret(string value);

AiCoreApi/SemanticKernel/Agents/BaseAgent.cs

@@ -14,7 +14,7 @@ public abstract class BaseAgent : IDoCallWrapperAgent
         private readonly ResponseAccessor _responseAccessor;
         private readonly RequestAccessor _requestAccessor;
         private readonly MonitoringConfig _monitoringConfig;
-        private readonly IIngestionParametersHelper _parametersHelper;
+        private readonly IParametersHelper _parametersHelper;
 
         private readonly ILogger<BaseAgent> _logger;
         private Dictionary<string, string>? _parameters;

AiCoreApi/SemanticKernel/Agents/BaseAgentHelper.cs

@@ -11,7 +11,7 @@ public BaseAgentHelper(
             RequestAccessor requestAccessor,
             ResponseAccessor responseAccessor,
             MonitoringConfig monitoringConfig,
-            IIngestionParametersHelper parametersHelper
+            IParametersHelper parametersHelper
             )
         {
             AgentsProcessor = agentsProcessor;
@@ -25,7 +25,7 @@ IIngestionParametersHelper parametersHelper
         public RequestAccessor RequestAccessor { get; }
         public ResponseAccessor ResponseAccessor { get; }
         public MonitoringConfig MonitoringConfig { get; }
-        public IIngestionParametersHelper ParametersHelper { get; }
+        public IParametersHelper ParametersHelper { get; }
     }
 
     public interface IBaseAgentHelper
@@ -34,6 +34,6 @@ public interface IBaseAgentHelper
         RequestAccessor RequestAccessor { get; }
         ResponseAccessor ResponseAccessor { get; }
         MonitoringConfig MonitoringConfig { get; }
-        IIngestionParametersHelper ParametersHelper { get; }
+        IParametersHelper ParametersHelper { get; }
     }
 }

AiCoreApi/SemanticKernel/Agents/GoogleSearchApiAgent.cs

@@ -6,6 +6,7 @@
 using System.Text.Json;
 using AiCoreApi.Common.Extensions;
 using System.Text.Encodings.Web;
+using Microsoft.Playwright;
 
 namespace AiCoreApi.SemanticKernel.Agents
 {
@@ -26,9 +27,10 @@ private static class AgentContentParameters
             public const string QueryString = "queryString";
             public const string GoogleConnection = "googleSearchApiConnection";
             public const string MaxContentLength = "maxContentLength";
-            public const string Count = "count"; 
-            public const string Offset = "offset"; 
+            public const string Count = "count";
+            public const string Offset = "offset";
             public const string OutputType = "outputType";
+            public const string WaitTimeout = "waitTimeout";
         }
 
         private const int DefaultMaxContentLength = 16384;
@@ -37,19 +39,22 @@ private static class AgentContentParameters
         private readonly ResponseAccessor _responseAccessor;
         private readonly IHttpClientFactory _httpClientFactory;
         private readonly IConnectionProcessor _connectionProcessor;
+        private readonly ExtendedConfig _extendedConfig;
 
         public GoogleSearchApiAgent(
             IBaseAgentHelper baseAgentHelper,
             RequestAccessor requestAccessor,
             ResponseAccessor responseAccessor,
             IHttpClientFactory httpClientFactory,
             IConnectionProcessor connectionProcessor,
+            ExtendedConfig extendedConfig,
             ILogger<GoogleSearchApiAgent> logger) : base(baseAgentHelper, logger)
         {
             _requestAccessor = requestAccessor;
             _responseAccessor = responseAccessor;
             _httpClientFactory = httpClientFactory;
             _connectionProcessor = connectionProcessor;
+            _extendedConfig = extendedConfig;
         }
 
         public override async Task<string> DoCall(AgentModel agent, Dictionary<string, string> parameters)
@@ -75,6 +80,7 @@ public override async Task<string> DoCall(AgentModel agent, Dictionary<string, s
             var count = int.Parse(await GetParameterValueAsync(AgentContentParameters.Count));
             var offset = int.Parse(await GetParameterValueAsync(AgentContentParameters.Offset));
             var outputType = agent.Content.TryGetValue(AgentContentParameters.OutputType, out var ot) ? ot.Value : "snippetTexts";
+            var waitTimeout = Convert.ToInt32(await GetParameterValueAsync(AgentContentParameters.WaitTimeout, "10000"));
             var results = await DoSearchAsync(queryString, googleConnection.Content["apiKey"], googleConnection.Content["googleCxId"], count, offset);
 
             string result;
@@ -92,13 +98,63 @@ public override async Task<string> DoCall(AgentModel agent, Dictionary<string, s
                     if (text.Length > int.Parse(maxContentLength))
                         text = text.Substring(0, int.Parse(maxContentLength));
 
-                    pages.Add(new Dictionary<string, string> { { "url", page.Url }, { "name", page.Name }, { "text", text } });
+                    pages.Add(new Dictionary<string, string>
+                    {
+                        { "url", page.Url },
+                        { "name", page.Name },
+                        { "text", text }
+                    });
                 }
-                result = JsonSerializer.Serialize(pages, new JsonSerializerOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping });
+
+                result = JsonSerializer.Serialize(pages,
+                    new JsonSerializerOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping });
+            }
+            else if (outputType == "pagesJsonPlaywright")
+            {
+                // NEW MODE � PLAYWRIGHT PAGE LOADING
+
+                PlaywrightInstall.EnsureInstalled();
+                using var playwright = await Playwright.CreateAsync();
+
+                await using var browser = await playwright.Chromium.LaunchAsync(new() { Headless = true });
+
+                await using var context = await browser.NewContextAsync(new BrowserNewContextOptions
+                {
+                    UserAgent = "Mozilla/5.0 (compatible; GoogleSearchApiAgent/Playwright)",
+                    Proxy = string.IsNullOrEmpty(_extendedConfig.Proxy)
+                        ? null
+                        : new Proxy { Server = _extendedConfig.Proxy },
+                    IgnoreHTTPSErrors = true
+                });
+
+                var pageObj = await context.NewPageAsync();
+
+                var pages = new List<Dictionary<string, string>>();
+
+                foreach (var page in results)
+                {
+                    var text = await CrawlWithPlaywrightAsync(page.Url, pageObj, waitTimeout);
+
+                    if (text.Length > int.Parse(maxContentLength))
+                        text = text.Substring(0, int.Parse(maxContentLength));
+
+                    pages.Add(new Dictionary<string, string>
+                    {
+                        { "url", page.Url },
+                        { "name", page.Name },
+                        { "text", text }
+                    });
+                }
+
+                result = JsonSerializer.Serialize(pages,
+                    new JsonSerializerOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping });
             }
             else
             {
-                result = JsonSerializer.Serialize(results.Select(r => r.Snippet).ToList(), new JsonSerializerOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping });
+                // snippetTexts � default
+                result = JsonSerializer.Serialize(
+                    results.Select(r => r.Snippet).ToList(),
+                    new JsonSerializerOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping });
             }
 
             _responseAccessor.AddDebugMessage(_debugMessageSenderName, "Execute Query String Result", result);
@@ -125,15 +181,21 @@ private async Task<List<WebPage>> DoSearchAsync(string query, string apiKey, str
             {
                 results.Add(new WebPage
                 {
-                    Name = item.GetProperty("title").GetString() ?? "",
-                    Url = item.GetProperty("link").GetString() ?? "",
-                    Snippet = item.GetProperty("snippet").GetString() ?? ""
+                    Name = SafeGet(item, "title"),
+                    Url = SafeGet(item, "link"),
+                    Snippet = SafeGet(item, "snippet")
                 });
             }
-
             return results;
         }
 
+        private static string SafeGet(JsonElement el, string propName)
+        {
+            return el.TryGetProperty(propName, out var p)
+                ? p.GetString() ?? ""
+                : "";
+        }
+
         private async Task<string> CrawlPageTextAsync(string url)
         {
             try
@@ -166,6 +228,44 @@ private async Task<string> CrawlPageTextAsync(string url)
             }
         }
 
+        private async Task<string> CrawlWithPlaywrightAsync(string url, IPage page, int waitTimeout)
+        {
+            try
+            {
+                await page.GotoAsync(url, new() { Timeout = waitTimeout });
+
+                var allText = new List<string>();
+
+                async Task ProcessFrame(IFrame frame)
+                {
+                    var content = await frame.ContentAsync();
+                    var doc = new HtmlDocument();
+                    doc.LoadHtml(content);
+
+                    doc.DocumentNode.Descendants()
+                        .Where(n => n.Name is "script" or "style")
+                        .ToList()
+                        .ForEach(n => n.Remove());
+
+                    var frameText = HtmlEntity.DeEntitize(doc.DocumentNode.InnerText);
+
+                    allText.AddRange(
+                        frameText.Split('\n')
+                        .Select(x => x.Trim())
+                        .Where(x => !string.IsNullOrWhiteSpace(x)));
+                }
+
+                foreach (var frame in page.Frames)
+                    await ProcessFrame(frame);
+
+                return string.Join("\n", allText);
+            }
+            catch (Exception ex)
+            {
+                _responseAccessor.AddDebugMessage(_debugMessageSenderName, "Playwright Error", $"Failed to read {url}, {ex.Message}");
+                return string.Empty;
+            }
+        }
 
         private async Task<HttpResponseMessage> SendGetRequestAsync(Uri uri, CancellationToken cancellationToken = default)
         {