Add P1 features: metrics, providers, datasets, integrations

agenteval-core/pom.xml

@@ -23,5 +23,10 @@
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+            <optional>true</optional>
+        </dependency>
     </dependencies>
 </project>

agenteval-core/src/main/java/com/agenteval/core/config/AgentEvalConfig.java

@@ -1,8 +1,11 @@
 package com.agenteval.core.config;
 
+import com.agenteval.core.cost.PricingModel;
 import com.agenteval.core.embedding.EmbeddingModel;
 import com.agenteval.core.judge.JudgeModel;
 
+import java.math.BigDecimal;
+
 /**
  * Configuration for the AgentEval evaluation engine.
  *
@@ -24,6 +27,8 @@ public final class AgentEvalConfig {
     private final boolean retryOnRateLimit;
     private final int maxRetries;
     private final boolean cacheJudgeResults;
+    private final BigDecimal costBudget;
+    private final PricingModel pricingModel;
 
     private AgentEvalConfig(Builder builder) {
         this.judgeModel = builder.judgeModel;
@@ -32,6 +37,8 @@ private AgentEvalConfig(Builder builder) {
         this.retryOnRateLimit = builder.retryOnRateLimit;
         this.maxRetries = builder.maxRetries;
         this.cacheJudgeResults = builder.cacheJudgeResults;
+        this.costBudget = builder.costBudget;
+        this.pricingModel = builder.pricingModel;
     }
 
     public JudgeModel judgeModel() { return judgeModel; }
@@ -40,6 +47,8 @@ private AgentEvalConfig(Builder builder) {
     public boolean retryOnRateLimit() { return retryOnRateLimit; }
     public int maxRetries() { return maxRetries; }
     public boolean cacheJudgeResults() { return cacheJudgeResults; }
+    public BigDecimal costBudget() { return costBudget; }
+    public PricingModel pricingModel() { return pricingModel; }
 
     public static Builder builder() {
         return new Builder();
@@ -59,6 +68,8 @@ public static final class Builder {
         private boolean retryOnRateLimit = true;
         private int maxRetries = 3;
         private boolean cacheJudgeResults = false;
+        private BigDecimal costBudget;
+        private PricingModel pricingModel;
 
         private Builder() {}
 
@@ -79,6 +90,8 @@ public Builder maxRetries(int maxRetries) {
             return this;
         }
         public Builder cacheJudgeResults(boolean cache) { this.cacheJudgeResults = cache; return this; }
+        public Builder costBudget(BigDecimal budget) { this.costBudget = budget; return this; }
+        public Builder pricingModel(PricingModel pricing) { this.pricingModel = pricing; return this; }
 
         public AgentEvalConfig build() {
             return new AgentEvalConfig(this);

agenteval-core/src/main/java/com/agenteval/core/config/AgentEvalConfigLoader.java

@@ -0,0 +1,120 @@
+package com.agenteval.core.config;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Loads {@link AgentEvalConfig} from an {@code agenteval.yaml} file.
+ *
+ * <p>Supports environment variable resolution in values using {@code ${ENV_VAR}} syntax.</p>
+ */
+public final class AgentEvalConfigLoader {
+
+    private static final Logger LOG = LoggerFactory.getLogger(AgentEvalConfigLoader.class);
+    private static final Pattern ENV_VAR = Pattern.compile("\\$\\{(\\w+)}");
+    private static final ObjectMapper YAML_MAPPER = new ObjectMapper(new YAMLFactory())
+            .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+    private AgentEvalConfigLoader() {}
+
+    /**
+     * Loads configuration from the given YAML file path.
+     *
+     * @param path the path to agenteval.yaml
+     * @return the populated config builder (call {@code .build()} to finalize)
+     */
+    public static AgentEvalConfig.Builder load(Path path) {
+        LOG.debug("Loading AgentEval config from {}", path);
+        try {
+            String content = Files.readString(path);
+            return parse(content);
+        } catch (IOException e) {
+            throw new ConfigException("Failed to load config from " + path, e);
+        }
+    }
+
+    /**
+     * Loads configuration from an input stream.
+     */
+    public static AgentEvalConfig.Builder load(InputStream inputStream) {
+        LOG.debug("Loading AgentEval config from input stream");
+        try {
+            String content = new String(inputStream.readAllBytes(),
+                    java.nio.charset.StandardCharsets.UTF_8);
+            return parse(content);
+        } catch (IOException e) {
+            throw new ConfigException("Failed to load config from input stream", e);
+        }
+    }
+
+    static AgentEvalConfig.Builder parse(String yamlContent) {
+        String resolved = resolveEnvVars(yamlContent);
+        try {
+            YamlConfigModel model = YAML_MAPPER.readValue(resolved, YamlConfigModel.class);
+            return toBuilder(model);
+        } catch (IOException e) {
+            throw new ConfigException("Failed to parse YAML config: " + e.getMessage(), e);
+        }
+    }
+
+    private static AgentEvalConfig.Builder toBuilder(YamlConfigModel model) {
+        var builder = AgentEvalConfig.builder();
+        if (model == null) {
+            return builder;
+        }
+
+        if (model.getDefaults() != null) {
+            var defaults = model.getDefaults();
+            if (defaults.getMaxRetries() != null) {
+                builder.maxRetries(defaults.getMaxRetries());
+            }
+            if (defaults.getRetryOnRateLimit() != null) {
+                builder.retryOnRateLimit(defaults.getRetryOnRateLimit());
+            }
+            if (defaults.getMaxConcurrentJudgeCalls() != null) {
+                builder.maxConcurrentJudgeCalls(defaults.getMaxConcurrentJudgeCalls());
+            }
+        }
+
+        if (model.getCost() != null && model.getCost().getBudget() != null) {
+            builder.costBudget(model.getCost().getBudget());
+        }
+
+        return builder;
+    }
+
+    static String resolveEnvVars(String content) {
+        Matcher matcher = ENV_VAR.matcher(content);
+        StringBuilder result = new StringBuilder();
+        while (matcher.find()) {
+            String envName = matcher.group(1);
+            String envValue = System.getenv(envName);
+            matcher.appendReplacement(result,
+                    Matcher.quoteReplacement(envValue != null ? envValue : ""));
+        }
+        matcher.appendTail(result);
+        return result.toString();
+    }
+
+    /**
+     * Returns the parsed {@link YamlConfigModel} for advanced use cases.
+     */
+    public static YamlConfigModel loadModel(Path path) {
+        try {
+            String content = resolveEnvVars(Files.readString(path));
+            return YAML_MAPPER.readValue(content, YamlConfigModel.class);
+        } catch (IOException e) {
+            throw new ConfigException("Failed to load config model from " + path, e);
+        }
+    }
+}

agenteval-core/src/main/java/com/agenteval/core/config/ConfigException.java

@@ -0,0 +1,17 @@
+package com.agenteval.core.config;
+
+/**
+ * Unchecked exception for configuration loading errors.
+ */
+public class ConfigException extends RuntimeException {
+
+    private static final long serialVersionUID = 1L;
+
+    public ConfigException(String message) {
+        super(message);
+    }
+
+    public ConfigException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java

@@ -0,0 +1,83 @@
+package com.agenteval.core.config;
+
+import java.math.BigDecimal;
+
+/**
+ * POJO representing the {@code agenteval.yaml} configuration file structure.
+ *
+ * <p>Used for Jackson YAML deserialization. Maps to {@link AgentEvalConfig}
+ * via {@link AgentEvalConfigLoader}.</p>
+ */
+public final class YamlConfigModel {
+
+    private JudgeSection judge;
+    private EmbeddingSection embedding;
+    private DefaultsSection defaults;
+    private CostSection cost;
+
+    public JudgeSection getJudge() { return judge; }
+    public void setJudge(JudgeSection judge) { this.judge = judge; }
+    public EmbeddingSection getEmbedding() { return embedding; }
+    public void setEmbedding(EmbeddingSection embedding) { this.embedding = embedding; }
+    public DefaultsSection getDefaults() { return defaults; }
+    public void setDefaults(DefaultsSection defaults) { this.defaults = defaults; }
+    public CostSection getCost() { return cost; }
+    public void setCost(CostSection cost) { this.cost = cost; }
+
+    public static final class JudgeSection {
+        private String provider;
+        private String model;
+        private String apiKey;
+        private String baseUrl;
+
+        public String getProvider() { return provider; }
+        public void setProvider(String provider) { this.provider = provider; }
+        public String getModel() { return model; }
+        public void setModel(String model) { this.model = model; }
+        public String getApiKey() { return apiKey; }
+        public void setApiKey(String apiKey) { this.apiKey = apiKey; }
+        public String getBaseUrl() { return baseUrl; }
+        public void setBaseUrl(String baseUrl) { this.baseUrl = baseUrl; }
+    }
+
+    public static final class EmbeddingSection {
+        private String provider;
+        private String model;
+        private String apiKey;
+        private String baseUrl;
+
+        public String getProvider() { return provider; }
+        public void setProvider(String provider) { this.provider = provider; }
+        public String getModel() { return model; }
+        public void setModel(String model) { this.model = model; }
+        public String getApiKey() { return apiKey; }
+        public void setApiKey(String apiKey) { this.apiKey = apiKey; }
+        public String getBaseUrl() { return baseUrl; }
+        public void setBaseUrl(String baseUrl) { this.baseUrl = baseUrl; }
+    }
+
+    public static final class DefaultsSection {
+        private Double threshold;
+        private Integer maxRetries;
+        private Boolean retryOnRateLimit;
+        private Integer maxConcurrentJudgeCalls;
+
+        public Double getThreshold() { return threshold; }
+        public void setThreshold(Double threshold) { this.threshold = threshold; }
+        public Integer getMaxRetries() { return maxRetries; }
+        public void setMaxRetries(Integer maxRetries) { this.maxRetries = maxRetries; }
+        public Boolean getRetryOnRateLimit() { return retryOnRateLimit; }
+        public void setRetryOnRateLimit(Boolean retryOnRateLimit) {
+            this.retryOnRateLimit = retryOnRateLimit;
+        }
+        public Integer getMaxConcurrentJudgeCalls() { return maxConcurrentJudgeCalls; }
+        public void setMaxConcurrentJudgeCalls(Integer max) { this.maxConcurrentJudgeCalls = max; }
+    }
+
+    public static final class CostSection {
+        private BigDecimal budget;
+
+        public BigDecimal getBudget() { return budget; }
+        public void setBudget(BigDecimal budget) { this.budget = budget; }
+    }
+}

agenteval-core/src/main/java/com/agenteval/core/cost/BudgetExceededException.java

@@ -0,0 +1,23 @@
+package com.agenteval.core.cost;
+
+import java.math.BigDecimal;
+
+/**
+ * Thrown when the cost budget has been exceeded.
+ */
+public class BudgetExceededException extends RuntimeException {
+
+    private static final long serialVersionUID = 1L;
+
+    private final BigDecimal currentCost;
+    private final BigDecimal budget;
+
+    public BudgetExceededException(BigDecimal currentCost, BigDecimal budget) {
+        super(String.format("Budget exceeded: $%s / $%s", currentCost, budget));
+        this.currentCost = currentCost;
+        this.budget = budget;
+    }
+
+    public BigDecimal getCurrentCost() { return currentCost; }
+    public BigDecimal getBudget() { return budget; }
+}

agenteval-core/src/main/java/com/agenteval/core/cost/CostSummary.java

@@ -0,0 +1,24 @@
+package com.agenteval.core.cost;
+
+import java.math.BigDecimal;
+
+/**
+ * Summary of accumulated costs from LLM judge calls.
+ *
+ * @param totalCost the total cost in USD
+ * @param totalInputTokens total input tokens consumed
+ * @param totalOutputTokens total output tokens consumed
+ */
+public record CostSummary(
+        BigDecimal totalCost,
+        long totalInputTokens,
+        long totalOutputTokens
+) {
+    public CostSummary {
+        if (totalCost == null) totalCost = BigDecimal.ZERO;
+    }
+
+    public long totalTokens() {
+        return totalInputTokens + totalOutputTokens;
+    }
+}