Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@ public class AgentRule

public class RuleConfig
{
[JsonPropertyName("topology_name")]
public string? TopologyName { get; set; }
[JsonPropertyName("criteria")]
public string? Criteria { get; set; }
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace BotSharp.Abstraction.Coding.Options;

public class CodeGenerationOptions : LlmConfigBase
public class CodeGenerationOptions
{
/// <summary>
/// Agent id to get instruction
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
namespace BotSharp.Abstraction.Rules.Constants;

/// <summary>
/// Built-in rule criteria types. Each value maps to an
/// <see cref="IRuleCriteriaEvaluator.Type"/> registered in DI.
/// Plugins may introduce additional string types.
/// </summary>
public static class BuiltInRuleCriteria
{
/// <summary>
/// Evaluate a code script (e.g. Python) that returns a boolean result.
/// </summary>
public const string Code = "code";

/// <summary>
/// Ask an LLM whether the rule applies to the request.
/// </summary>
public const string Llm = "llm";
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
using BotSharp.Abstraction.Rules.Models;

namespace BotSharp.Abstraction.Rules;

/// <summary>
/// Decides whether a rule should be executed for the current request.
/// Implementations are resolved by <see cref="Type"/> in the rule engine,
/// so new criteria mechanisms can be added without changing the engine.
/// </summary>
public interface IRuleCriteriaEvaluator
{
/// <summary>
/// The criteria type this evaluator handles
/// </summary>
string Type { get; }

/// <summary>
/// Evaluate the criteria for a single agent's rule.
/// </summary>
/// <param name="agent">The agent whose rule is being considered</param>
/// <param name="trigger">The rule trigger</param>
/// <param name="context">The per-request criteria context</param>
/// <returns>True if the rule should be executed for this request.</returns>
Task<bool> EvaluateAsync(Agent agent, IRuleTrigger trigger, RuleCriteriaContext context);
}
23 changes: 10 additions & 13 deletions src/Infrastructure/BotSharp.Abstraction/Rules/IRuleEngine.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
using BotSharp.Abstraction.Graph;
using BotSharp.Abstraction.Rules.Models;

namespace BotSharp.Abstraction.Rules;

public interface IRuleEngine
Expand All @@ -17,14 +14,14 @@ public interface IRuleEngine
Task<IEnumerable<string>> Triggered(IRuleTrigger trigger, string text, IEnumerable<MessageState>? states = null, RuleTriggerOptions? options = null)
=> throw new NotImplementedException();

/// <summary>
/// Execute rule graph node
/// </summary>
/// <param name="node"></param>
/// <param name="graph"></param>
/// <param name="agentId"></param>
/// <param name="trigger"></param>
/// <param name="options"></param>
/// <returns></returns>
Task ExecuteGraphNode(FlowNode node, FlowGraph graph, string agentId, IRuleTrigger trigger, RuleNodeExecutionOptions options);
///// <summary>
///// Execute rule graph node
///// </summary>
///// <param name="node"></param>
///// <param name="graph"></param>
///// <param name="agentId"></param>
///// <param name="trigger"></param>
///// <param name="options"></param>
///// <returns></returns>
//Task ExecuteGraphNode(FlowNode node, FlowGraph graph, string agentId, IRuleTrigger trigger, RuleNodeExecutionOptions options);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
namespace BotSharp.Abstraction.Rules.Models;

/// <summary>
/// The per-request context passed to an <see cref="IRuleCriteriaEvaluator"/>
/// to decide whether a rule should be executed.
/// </summary>
public class RuleCriteriaContext
{
/// <summary>
/// The trigger message text.
/// </summary>
public string Text { get; set; } = string.Empty;

/// <summary>
/// The criteria options (evaluator type and its arguments).
/// </summary>
public CriteriaOptions Options { get; set; } = new();

/// <summary>
/// The conversation states carried with the request.
/// </summary>
public IEnumerable<MessageState>? States { get; set; }
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using BotSharp.Abstraction.Repositories.Filters;
using BotSharp.Abstraction.Rules.Constants;
using System.Text.Json;

namespace BotSharp.Abstraction.Rules.Options;
Expand All @@ -11,12 +12,39 @@ public class RuleTriggerOptions
public AgentFilter? AgentFilter { get; set; }

/// <summary>
/// Json serializer options
/// Criteria
/// </summary>
public JsonSerializerOptions? JsonOptions { get; set; }
public CriteriaOptions? Criteria { get; set; }
}

public class CriteriaOptions
{
/// <summary>
/// Rule flow options
/// How the criteria is evaluated (see <see cref="BuiltInRuleCriteria"/>).
/// Selects which <c>IRuleCriteriaEvaluator</c> handles this criteria.
/// </summary>
public RuleFlowOptions? Flow { get; set; }
}
public string Type { get; set; } = BuiltInRuleCriteria.Code;

/// <summary>
/// Evaluator-specific settings, kept as raw JSON so each evaluator can
/// deserialize it into its own strongly-typed settings model.
/// Use <see cref="GetData{T}"/> to read it.
/// </summary>
public JsonElement? Data { get; set; }

/// <summary>
/// Deserialize <see cref="Data"/> into an evaluator-specific settings type.
/// Returns default (null) when no data is provided.
/// </summary>
public T? GetData<T>(JsonSerializerOptions? options = null)
{
if (Data == null || Data.Value.ValueKind == JsonValueKind.Null || Data.Value.ValueKind == JsonValueKind.Undefined)
{
return default;
}

return Data.Value.Deserialize<T>(options ?? _webJsonOptions);
}

private static readonly JsonSerializerOptions _webJsonOptions = new(JsonSerializerDefaults.Web);
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
namespace BotSharp.Core.Rules.Criteria.Code;

/// <summary>
/// Evaluates rule trigger criteria by running an agent code script (e.g. Python)
/// that returns a boolean ("true"/"false") result.
/// </summary>
public class CodeCriteriaEvaluator : IRuleCriteriaEvaluator
{
private readonly IServiceProvider _services;
private readonly ILogger<CodeCriteriaEvaluator> _logger;
private readonly CodingSettings _codingSettings;

public CodeCriteriaEvaluator(
IServiceProvider services,
ILogger<CodeCriteriaEvaluator> logger,
CodingSettings codingSettings)
{
_services = services;
_logger = logger;
_codingSettings = codingSettings;
}

public string Type => BuiltInRuleCriteria.Code;

public async Task<bool> EvaluateAsync(Agent agent, IRuleTrigger trigger, RuleCriteriaContext context)
{
var settings = context.Options.GetData<CodeCriteriaSettings>() ?? new();
var provider = settings.CodeProcessor ?? BuiltInCodeProcessor.PyInterpreter;
var processor = _services.GetServices<ICodeProcessor>().FirstOrDefault(x => x.Provider.IsEqualTo(provider));
if (processor == null)
{
_logger.LogWarning($"Unable to find code processor: {provider}.");
return true;
}

var agentService = _services.GetRequiredService<IAgentService>();
var scriptName = settings.CodeScriptName ?? $"{trigger.Name}_rule.py";
var codeScript = await agentService.GetAgentCodeScript(agent.Id, scriptName, scriptType: AgentCodeScriptType.Src);

var msg = $"rule trigger ({trigger.Name}) code script ({scriptName}) in agent ({agent.Name}) => args: {settings.ArgumentContent?.RootElement.GetRawText()}.";

if (codeScript == null || string.IsNullOrWhiteSpace(codeScript.Content))
{
_logger.LogWarning($"Unable to find {msg}.");
return true;
}
Comment on lines +27 to +46

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Action required

1. codecriteriaevaluator fails open 📘 Rule violation ☼ Reliability

CodeCriteriaEvaluator.EvaluateAsync returns true when it cannot evaluate criteria (missing code
processor or missing script), causing rules to execute even though criteria validation failed. This
violates the requirement to validate inputs and fail safely at integration boundaries.
Agent Prompt
## Issue description
`CodeCriteriaEvaluator.EvaluateAsync` currently returns `true` when the evaluator cannot run (e.g., no `ICodeProcessor` found or no script content). This is a fail-open behavior that can trigger rules when criteria evaluation is effectively unavailable.

## Issue Context
Compliance requires validating boundary inputs/dependencies and providing safe failure behavior when required data/services are missing or invalid.

## Fix Focus Areas
- src/Infrastructure/BotSharp.Core.Rules/Criteria/Code/CodeCriteriaEvaluator.cs[27-46]

ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools


try
{
var hooks = _services.GetHooks<IInstructHook>(agent.Id);

var arguments = BuildArguments(settings.ArgumentName, settings.ArgumentContent);
var codeContext = new CodeExecutionContext
{
CodeScript = codeScript,
Arguments = arguments
};

foreach (var hook in hooks)
{
await hook.BeforeCodeExecution(agent, codeContext);
}

var (useLock, useProcess, timeoutSeconds) = CodingUtil.GetCodeExecutionConfig(_codingSettings);
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds));
var response = processor.Run(codeScript.Content, options: new()
{
ScriptName = scriptName,
Arguments = arguments,
UseLock = useLock,
UseProcess = useProcess
}, cancellationToken: cts.Token);

var codeResponse = new CodeExecutionResponseModel
{
CodeProcessor = processor.Provider,
CodeScript = codeScript,
Arguments = arguments.DistinctBy(x => x.Key).ToDictionary(x => x.Key, x => x.Value ?? string.Empty),
ExecutionResult = response
};

foreach (var hook in hooks)
{
await hook.AfterCodeExecution(agent, codeContext, codeResponse);
}

if (response == null || !response.Success)
{
_logger.LogWarning($"Failed to handle {msg}");
return false;
}

bool result;
LogLevel logLevel;
if (response.Result.IsEqualTo("true"))
{
logLevel = LogLevel.Information;
result = true;
}
else
{
logLevel = LogLevel.Warning;
result = false;
}

_logger.Log(logLevel, $"Code script execution result ({response}) from {msg}");
return result;
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error when handling {msg}");
return false;
}
}

private List<KeyValue> BuildArguments(string? name, JsonDocument? args)
{
var keyValues = new List<KeyValue>();
if (args != null)
{
keyValues.Add(new KeyValue(name ?? "trigger_args", args.RootElement.GetRawText()));
}
return keyValues;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
using System.Text.Json.Serialization;

namespace BotSharp.Core.Rules.Criteria.Code;

/// <summary>
/// Settings for <see cref="CodeCriteriaEvaluator"/>, parsed from
/// <c>CriteriaOptions.Data</c>.
/// </summary>
public class CodeCriteriaSettings
{
/// <summary>
/// Code processor provider (defaults to the Python interpreter).
/// </summary>
[JsonPropertyName("code_processor")]
public string? CodeProcessor { get; set; }

/// <summary>
/// Code script name.
/// </summary>
[JsonPropertyName("code_script_name")]
public string? CodeScriptName { get; set; }

/// <summary>
/// Argument name as an input key to the code script.
/// </summary>
[JsonPropertyName("argument_name")]
public string? ArgumentName { get; set; }

/// <summary>
/// Json arguments as an input value to the code script.
/// </summary>
[JsonPropertyName("argument_content")]
public JsonDocument? ArgumentContent { get; set; }
}
Loading
Loading