Add P3 ecosystem: multi-judge, versioning, Maven plugin, GitHub Actions

agenteval-core/src/main/java/com/agenteval/core/config/YamlConfigModel.java

@@ -1,6 +1,7 @@
 package com.agenteval.core.config;
 
 import java.math.BigDecimal;
+import java.util.List;
 
 /**
  * POJO representing the {@code agenteval.yaml} configuration file structure.
@@ -29,6 +30,8 @@ public static final class JudgeSection {
         private String model;
         private String apiKey;
         private String baseUrl;
+        private List<JudgeSection> models;
+        private String consensusStrategy;
 
         public String getProvider() { return provider; }
         public void setProvider(String provider) { this.provider = provider; }
@@ -38,6 +41,12 @@ public static final class JudgeSection {
         public void setApiKey(String apiKey) { this.apiKey = apiKey; }
         public String getBaseUrl() { return baseUrl; }
         public void setBaseUrl(String baseUrl) { this.baseUrl = baseUrl; }
+        public List<JudgeSection> getModels() { return models; }
+        public void setModels(List<JudgeSection> models) { this.models = models; }
+        public String getConsensusStrategy() { return consensusStrategy; }
+        public void setConsensusStrategy(String consensusStrategy) {
+            this.consensusStrategy = consensusStrategy;
+        }
     }
 
     public static final class EmbeddingSection {

agenteval-datasets/pom.xml

@@ -24,6 +24,14 @@
             <artifactId>agenteval-judge</artifactId>
             <optional>true</optional>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>

agenteval-datasets/src/main/java/com/agenteval/datasets/EvalDataset.java

@@ -4,6 +4,8 @@
 import com.agenteval.datasets.csv.CsvDatasetWriter;
 import com.agenteval.datasets.json.JsonDatasetWriter;
 import com.agenteval.datasets.jsonl.JsonlDatasetWriter;
+import com.agenteval.datasets.version.DatasetVersioner;
+import com.agenteval.datasets.version.VersionedDataset;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
 
@@ -72,6 +74,17 @@ public void save(Path path, DatasetFormat format) {
         }
     }
 
+    /**
+     * Tags this dataset with a version label and saves it to the storage directory.
+     *
+     * @param label      the version label (e.g., "v1.0")
+     * @param storageDir the directory where versioned datasets are stored
+     * @return the versioned dataset with git metadata
+     */
+    public VersionedDataset tagVersion(String label, Path storageDir) {
+        return new DatasetVersioner(storageDir).tag(this, label);
+    }
+
     @JsonPOJOBuilder(withPrefix = "")
     public static final class Builder {
         private String name;

agenteval-datasets/src/main/java/com/agenteval/datasets/version/DatasetVersion.java

@@ -0,0 +1,29 @@
+package com.agenteval.datasets.version;
+
+import java.time.Instant;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Version metadata for a golden dataset.
+ *
+ * @param versionLabel the human-readable version label (e.g., "v1.0", "2024-03-baseline")
+ * @param gitMetadata  git repository state at version time (may be null)
+ * @param createdAt    when this version was created
+ * @param extra        arbitrary additional metadata
+ */
+public record DatasetVersion(
+        String versionLabel,
+        GitMetadata gitMetadata,
+        Instant createdAt,
+        Map<String, Object> extra
+) {
+
+    public DatasetVersion {
+        Objects.requireNonNull(versionLabel, "versionLabel must not be null");
+        if (createdAt == null) {
+            createdAt = Instant.now();
+        }
+        extra = extra == null ? Map.of() : Map.copyOf(extra);
+    }
+}

agenteval-datasets/src/main/java/com/agenteval/datasets/version/DatasetVersioner.java

@@ -0,0 +1,164 @@
+package com.agenteval.datasets.version;
+
+import com.agenteval.datasets.DatasetException;
+import com.agenteval.datasets.EvalDataset;
+import com.agenteval.datasets.json.JsonDatasetLoader;
+import com.agenteval.datasets.json.JsonDatasetWriter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Stream;
+
+/**
+ * Manages versioned golden datasets with git metadata.
+ *
+ * <p>Stores versioned datasets in a directory structure:
+ * {@code <storageDir>/<datasetName>/<versionLabel>/dataset.json} alongside
+ * a {@code version.json} metadata file.</p>
+ */
+public final class DatasetVersioner {
+
+    private static final Logger LOG = LoggerFactory.getLogger(DatasetVersioner.class);
+    private static final String DATASET_FILE = "dataset.json";
+    private static final String VERSION_FILE = "version.json";
+    private static final ObjectMapper MAPPER = new ObjectMapper()
+            .registerModule(new JavaTimeModule())
+            .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
+            .enable(SerializationFeature.INDENT_OUTPUT);
+
+    private final Path storageDir;
+    private final GitResolver gitResolver;
+
+    /**
+     * Creates a versioner that stores datasets under the given directory
+     * and resolves git metadata from the given working directory.
+     */
+    public DatasetVersioner(Path storageDir, Path gitWorkingDir) {
+        this.storageDir = Objects.requireNonNull(storageDir, "storageDir must not be null");
+        this.gitResolver = new GitResolver(
+                Objects.requireNonNull(gitWorkingDir, "gitWorkingDir must not be null"));
+    }
+
+    /**
+     * Creates a versioner that stores datasets under the given directory.
+     * Git metadata is resolved from the storage directory itself.
+     */
+    public DatasetVersioner(Path storageDir) {
+        this(storageDir, storageDir);
+    }
+
+    /**
+     * Tags a dataset with a version label, resolves git metadata, and saves it.
+     *
+     * @param dataset the dataset to version
+     * @param label   the version label (e.g., "v1.0")
+     * @return the versioned dataset
+     */
+    public VersionedDataset tag(EvalDataset dataset, String label) {
+        Objects.requireNonNull(dataset, "dataset must not be null");
+        Objects.requireNonNull(label, "label must not be null");
+
+        String name = dataset.getName();
+        if (name == null || name.isBlank()) {
+            throw new DatasetException("Dataset must have a name for versioning");
+        }
+
+        GitMetadata git = gitResolver.resolve();
+        DatasetVersion version = new DatasetVersion(label, git, Instant.now(), null);
+
+        Path versionDir = storageDir.resolve(name).resolve(label);
+        try {
+            Files.createDirectories(versionDir);
+            new JsonDatasetWriter().write(dataset, versionDir.resolve(DATASET_FILE));
+            MAPPER.writeValue(versionDir.resolve(VERSION_FILE).toFile(), version);
+        } catch (IOException e) {
+            throw new DatasetException("Failed to save versioned dataset: " + e.getMessage(), e);
+        }
+
+        LOG.info("Tagged dataset '{}' as version '{}'", name, label);
+        return new VersionedDataset(dataset, version);
+    }
+
+    /**
+     * Loads a specific version of a dataset.
+     *
+     * @param name  the dataset name
+     * @param label the version label
+     * @return the versioned dataset
+     */
+    public VersionedDataset load(String name, String label) {
+        Path versionDir = storageDir.resolve(name).resolve(label);
+        Path datasetFile = versionDir.resolve(DATASET_FILE);
+        Path versionFile = versionDir.resolve(VERSION_FILE);
+
+        if (!Files.exists(datasetFile)) {
+            throw new DatasetException(
+                    "Version '" + label + "' not found for dataset '" + name + "'");
+        }
+
+        EvalDataset dataset = new JsonDatasetLoader().load(datasetFile);
+        DatasetVersion version;
+        try {
+            version = MAPPER.readValue(versionFile.toFile(), DatasetVersion.class);
+        } catch (IOException e) {
+            throw new DatasetException("Failed to read version metadata: " + e.getMessage(), e);
+        }
+
+        return new VersionedDataset(dataset, version);
+    }
+
+    /**
+     * Lists all version labels for a dataset, sorted by creation time (newest first).
+     *
+     * @param name the dataset name
+     * @return the list of version labels
+     */
+    public List<String> listVersions(String name) {
+        Path datasetDir = storageDir.resolve(name);
+        if (!Files.isDirectory(datasetDir)) {
+            return List.of();
+        }
+
+        try (Stream<Path> dirs = Files.list(datasetDir)) {
+            return dirs
+                    .filter(Files::isDirectory)
+                    .filter(d -> Files.exists(d.resolve(VERSION_FILE)))
+                    .sorted(Comparator.<Path, Long>comparing(d -> {
+                        try {
+                            return Files.getLastModifiedTime(d.resolve(VERSION_FILE))
+                                    .toMillis();
+                        } catch (IOException e) {
+                            return 0L;
+                        }
+                    }).reversed())
+                    .map(d -> d.getFileName().toString())
+                    .toList();
+        } catch (IOException e) {
+            throw new DatasetException("Failed to list versions: " + e.getMessage(), e);
+        }
+    }
+
+    /**
+     * Loads the latest (most recently created) version of a dataset.
+     *
+     * @param name the dataset name
+     * @return the latest versioned dataset
+     */
+    public VersionedDataset latest(String name) {
+        List<String> versions = listVersions(name);
+        if (versions.isEmpty()) {
+            throw new DatasetException("No versions found for dataset '" + name + "'");
+        }
+        return load(name, versions.getFirst());
+    }
+}

agenteval-datasets/src/main/java/com/agenteval/datasets/version/GitMetadata.java

@@ -0,0 +1,24 @@
+package com.agenteval.datasets.version;
+
+import java.time.Instant;
+
+/**
+ * Git repository metadata captured at the time of dataset versioning.
+ *
+ * @param commitHash  full commit SHA
+ * @param shortHash   abbreviated commit SHA
+ * @param branch      current branch name (may be null for detached HEAD)
+ * @param tag         most recent tag reachable from HEAD (may be null)
+ * @param timestamp   commit timestamp
+ * @param authorName  commit author name
+ * @param authorEmail commit author email
+ */
+public record GitMetadata(
+        String commitHash,
+        String shortHash,
+        String branch,
+        String tag,
+        Instant timestamp,
+        String authorName,
+        String authorEmail
+) {}