From 7296d8ed3b600025dd9182bcc3d75c4a635203fc Mon Sep 17 00:00:00 2001 From: Eric Deandrea Date: Mon, 2 Mar 2026 15:13:26 -0500 Subject: [PATCH] docs: Add docs for version tester Signed-off-by: Eric Deandrea --- .../docling-version-tests/README.md | 313 ++++++++++++++++++ .../docling-version-tests/build.gradle.kts | 4 +- .../images/architecture.puml | 116 +++++++ .../client/tester/service/TagsTester.java | 7 +- 4 files changed, 434 insertions(+), 6 deletions(-) create mode 100644 docling-testing/docling-version-tests/README.md create mode 100644 docling-testing/docling-version-tests/images/architecture.puml diff --git a/docling-testing/docling-version-tests/README.md b/docling-testing/docling-version-tests/README.md new file mode 100644 index 00000000..64d11aa2 --- /dev/null +++ b/docling-testing/docling-version-tests/README.md @@ -0,0 +1,313 @@ +# Docling Version Tests + +A [Quarkus](https://quarkus.io)-based command-line application for automated compatibility testing of the [`docling-serve-api`](../../docling-serve/docling-serve-api) client library against multiple versions of the [Docling Serve](https://github.com/docling-project/docling-serve) container image. + +## Overview + +This application automatically tests `docling-java` client compatibility by: + +1. Fetching available Docker image tags from the [Docling Serve container registry](https://ghcr.io/docling-project/docling-serve) +2. Starting Docling Serve containers for each version tag +3. Executing conversion requests against each container +4. Validating responses and health checks +5. Generating detailed markdown reports +6. Optionally creating GitHub issues for failures + +## Technologies + +This application leverages the following frameworks and libraries: + +### Core Framework + +- **[Quarkus](https://quarkus.io)** - Supersonic Subatomic Java framework for building cloud-native applications + +### Quarkus Extensions + +| Extension | Purpose | Documentation | +|-----------------------------------------------------------------------------|----------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------| +| [Picocli](https://quarkus.io/guides/picocli) | Command-line interface framework with argument parsing and help generation | [Quarkus Picocli Guide](https://quarkus.io/guides/picocli) | +| [Config YAML](https://quarkus.io/guides/config-yaml) | YAML-based configuration support (`application.yml`) | [Quarkus YAML Configuration](https://quarkus.io/guides/config-yaml) | +| [ArC](https://quarkus.io/guides/cdi-reference) | Dependency injection (CDI implementation) | [Quarkus CDI Reference](https://quarkus.io/guides/cdi-reference) | +| [SmallRye Config](https://quarkus.io/guides/config-reference) | Type-safe configuration mapping (`@ConfigMapping`) | [Quarkus Configuration Guide](https://quarkus.io/guides/config-reference) | +| [REST Client Jackson](https://quarkus.io/guides/rest-client) | Reactive REST client with Jackson serialization (for registry API calls) | [Quarkus REST Client Guide](https://quarkus.io/guides/rest-client) | +| [Qute](https://quarkus.io/guides/qute) | Server-side template engine (generates markdown reports) | [Quarkus Qute Guide](https://quarkus.io/guides/qute) | +| [JUnit 5](https://quarkus.io/guides/getting-started-testing) | Testing framework with Quarkus extensions | [Quarkus Testing Guide](https://quarkus.io/guides/getting-started-testing) | +| [WireMock](https://docs.quarkiverse.io/quarkus-wiremock/dev/index.html) | HTTP mocking for integration tests | [Quarkus WireMock Extension](https://docs.quarkiverse.io/quarkus-wiremock/dev/index.html) | +| [GitHub API](https://docs.quarkiverse.io/quarkus-github-api/dev/index.html) | GitHub REST API client (for automated issue creation) | [Quarkus GitHub API Extension](https://docs.quarkiverse.io/quarkus-github-api/dev/index.html) | + +### Additional Libraries + +- **[AssertJ](https://assertj.github.io/doc/)** - Fluent assertions for validation and testing +- **[Docling Testcontainers](https://testcontainers.com/modules/docling/)** - Spins up ephemeral Docling Serve Docker containers for each version test +- **[Semver4j](https://github.com/vdurmont/semver4j)** - Semantic versioning parsing and filtering +- **[SmallRye Mutiny](https://smallrye.io/smallrye-mutiny/)** - Reactive programming for parallel test execution + +### Internal Dependencies + +- [`docling-serve-client`](../../docling-serve/docling-serve-client) - Java HTTP client for Docling Serve API +- [`docling-testcontainers`](../../docling-testcontainers) - Testcontainers wrapper for Docling Serve +- [`docling-core`](../../docling-core) - Core Docling document model + +## How It Works + +### Architecture + +![Architecture](https://www.plantuml.com/plantuml/proxy?cache=no&src=https://raw.githubusercontent.com/docling-project/docling-java/main/docling-testing/docling-version-tests/images/architecture.puml) + +### Execution Flow + +#### 1. Command Initialization (`VersionTestsCommand`) + +The application starts via [Quarkus Picocli](https://quarkus.io/guides/picocli) integration. The [`VersionTestsCommand`](src/main/java/ai/docling/client/tester/VersionTestsCommand.java) class is annotated with `@Command` and implements `Runnable`. It defines CLI options: + +- `-p, --parallelism`: Number of concurrent container tests (default: 1) +- `-i, --image`: Docker image to test (default: `docling-project/docling-serve`) +- `-r, --registry`: Container registry URL (default: `ghcr.io`) +- `-o, --output`: Results output directory (default: `results/`) +- `-t, --tags`: Explicit list of tags to test (fetches all if omitted) +- `-e, --exclude-tags-regex`: Regex pattern to exclude tags +- `-g, --create-github-issue`: Create GitHub issue on test failure +- `-c, --cleanup-container-images`: Remove Docker images after testing + +#### 2. Tag Discovery (`GHCRClient`) + +If `--tags` is not provided, the application queries the container registry: + +1. **Authentication**: Calls the registry's token endpoint (`/token?scope=repository:{image}:pull`) +2. **Fetch Tags**: Paginates through `/v2/{image}/tags/list` (1000 tags per page) +3. **Filter Versions**: Extracts semantic version tags (e.g., `v1.13.0`) and excludes non-version tags +4. **Apply Exclusions**: Filters out tags matching `--exclude-tags-regex` + +The `GHCRClient` is a Quarkus REST Client interface (`@RegisterRestClient`) with declarative HTTP methods. + +#### 3. Parallel Testing (`TagsTester` + `WorkParallelizer`) + +For each tag, the `TagsTester` service: + +1. **Spin Up Container**: Creates a `DoclingServeContainer` (Testcontainers wrapper) for the specific tag + ```java + var containerConfig = DoclingServeContainerConfig.builder() + .image("{registry}/{image}:{tag}") + .startupTimeout(Duration.ofMinutes(5)) + .build(); + var doclingContainer = new DoclingServeContainer(containerConfig); + doclingContainer.start(); + ``` + +2. **Health Check**: Validates the `/health` endpoint returns `{"status": "ok"}` + +3. **Execute Conversion**: Sends a test conversion request + - **Source**: `https://docling.ai` (via `HttpSource`) + - **Output Formats**: Markdown, JSON, Text + - **Options**: `abortOnError=true`, `includeImages=true` + +4. **Assertions**: Uses AssertJ to validate: + - Response is not null + - No errors in response + - All output formats (markdown, text, JSON) are non-empty + - JSON content deserializes to `DoclingDocument` + +5. **Capture Logs**: Retrieves container logs via `doclingContainer.getLogs()` + +6. **Cleanup**: Optionally removes the Docker image to save disk space + +7. **Record Result**: Stores outcome as `TagTestResult` (success or failure with stack trace) + +Parallelization is achieved via **SmallRye Mutiny** (`Multi.transformToUniAndMerge`), distributing work across a fixed thread pool. + +#### 4. Results Handling + +Results are processed by two handlers (run in parallel via `WorkParallelizer`): + +##### A. [**MarkdownFileResultsHandler**](src/main/java/ai/docling/client/tester/service/results/MarkdownFileResultsHandler.java) + +Generates a comprehensive markdown report using **Qute templates**: + +- **Template**: [`results.md`](src/main/resources/templates/results/results.md) +- **Output**: `{output-dir}/results-{timestamp}/results.md` +- **Content**: + - Summary table with ✅/❌ icons per tag + - Expandable detail sections for each tag: + - Success/failure message + - Full stack trace (if failed) + - Container logs + +##### B. [**GithubIssueResultsHandler**](src/main/java/ai/docling/client/tester/service/results/GithubIssueResultsHandler.java) + +If `--create-github-issue` is enabled and at least one test fails: + +1. Reads `GITHUB_TOKEN` from environment +2. Creates an issue in the configured repository (default: `docling-project/docling-java`) +3. Issue body contains the markdown report +4. Labels: `automation`, `area:docling-serve` + +Configured via `@ConfigMapping`: +```yaml +docling-version-tester: + github: + issue-creation: + issue-org: docling-project # default + issue-repo: docling-java # default +``` + +### Configuration + +The application uses **Quarkus Config** with YAML support ([`application.yml`](src/main/resources/application.yml)): + +- **REST Client**: Configures the `github-container-registry` client base URL (`https://ghcr.io`) +- **WireMock**: Dev/test mode uses WireMock to mock registry responses +- **Logging**: Configures log levels for Testcontainers, Docker client, etc. +- **Profiles**: + - `%dev,test`: Uses WireMock on `localhost` + - `%no-wiremock`: Disables WireMock for real registry calls + +### Testing + +The application includes unit tests for: + +- **Command Options**: Validates Picocli argument parsing ([`VersionTestsCommandOptionTests`](src/test/java/ai/docling/client/tester/VersionTestsCommandOptionTests.java)) +- **Domain Models**: Tests `TagTestResult` serialization ([`TagTestResultTests`](src/test/java/ai/docling/client/tester/domain/TagTestResultTests.java)) +- **GHCR Client**: Mocks registry API with WireMock ([`GHCRClientTests`](src/test/java/ai/docling/client/tester/client/ghcr/GHCRClientTests.java)) + +Tests use: +- [**Quarkus JUnit**](https://quarkus.io/guides/getting-started-testing) (`@QuarkusTest`) +- [**Quarkus WireMock**](https://docs.quarkiverse.io/quarkus-wiremock/dev/index.html) for HTTP mocking +- [**AssertJ**](https://assertj.github.io/doc/) for fluent assertions + +## Usage + +### Development Mode (with live reload) + +Run the CLI with specific arguments: +```bash +# Test specific tags +./gradlew :docling-version-tests:quarkusDev -Dquarkus.args="-t v1.13.0 v1.12.0 -p 2" + +# Test all tags from GHCR +./gradlew :docling-version-tests:quarkusDev -Dquarkus.args="-p 4 --exclude-tags-regex '.*rc.*'" +``` + +### CI/CD Integration + +GitHub Actions workflow ([`version-tests.yml`](../../.github/workflows/version-tests.yml)) runs this application on a schedule: + +```yaml +- name: Run version tests + run: | + ./gradlew :docling-version-tests:build + java -jar docling-testing/docling-version-tests/build/quarkus-app/quarkus-run.jar \ + --parallelism 2 \ + --output results \ + --create-github-issue + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +``` + +## Requirements + +- **Java 17+** (tested on 17, 21, 25) +- **Docker** or **Podman** (for Testcontainers) +- **Gradle 8.5+** +- **Optional**: GraalVM for native compilation +- **Optional**: `GITHUB_TOKEN` environment variable (for issue creation) + +## Output + +Results are written to `{output-dir}/results-{timestamp}/`: + +``` +results/ +└── results-2026-03-02T14-30-00.123456Z/ + └── results.md +``` + +Example `results.md` excerpt: + +```markdown +# Results for ghcr.io/docling-project/docling-serve as of 2026-03-02T14:30:00.123456Z + +| Tag | Result | Details | +| --- | ------ | ------- | +| v1.13.0 | ✅ SUCCESS | [Click for run details](#v1.13.0-details) | +| v1.12.0 | ❌ FAILURE | [Click for run details](#v1.12.0-details) | +``` + +## Development + +### Project Structure + +``` +docling-version-tests/ +├── src/ +│ ├── main/ +│ │ ├── java/ai/docling/client/tester/ +│ │ │ ├── VersionTestsCommand.java # CLI entrypoint +│ │ │ ├── client/ +│ │ │ │ ├── RegistryClient.java # Registry API interface +│ │ │ │ ├── RegistryClientFactory.java # Client factory +│ │ │ │ └── ghcr/ +│ │ │ │ └── GHCRClient.java # GHCR REST client +│ │ │ ├── config/ +│ │ │ │ └── Config.java # Type-safe config +│ │ │ ├── domain/ +│ │ │ │ ├── Tags.java # Tag domain model +│ │ │ │ ├── TagsTestRequest.java # Test request +│ │ │ │ ├── TagsTestResults.java # Test results +│ │ │ │ └── TagTestResult.java # Single tag result +│ │ │ └── service/ +│ │ │ ├── TagsTester.java # Core testing logic +│ │ │ ├── WorkParallelizer.java # Parallel execution +│ │ │ └── results/ +│ │ │ ├── ResultsHandler.java # Handler interface +│ │ │ ├── ResultsHandlers.java # Handler aggregator +│ │ │ ├── MarkdownFileResultsHandler.java +│ │ │ └── GithubIssueResultsHandler.java +│ │ └── resources/ +│ │ ├── application.yml # Quarkus config +│ │ └── templates/ +│ │ └── results/ +│ │ └── results.md # Qute template +│ └── test/ +│ ├── java/ # Unit tests +│ └── resources/ +│ └── wiremock/ # WireMock stubs +└── build.gradle.kts +``` + +### Adding a New Registry Client + +1. Create interface extending `RegistryClient` in `client/` +2. Annotate with `@RegisterRestClient(configKey = "my-registry")` +3. Implement `getTokenForImage` and `getTags` methods +4. Add configuration to `application.yml`: + ```yaml + quarkus: + rest-client: + my-registry: + url: https://my-registry.io + ``` +5. Update `RegistryClientFactory` to return your client + +### Customizing the Report + +Edit the Qute template at `src/main/resources/templates/results/results.md`. Qute syntax: + +```markdown +{#each results.results} + Tag: {it.tag}, Status: {it.result.status.name()} +{/each} +``` + +## License + +This application is part of the `docling-java` project. See [LICENSE](../../LICENSE) for details. + +## Contributing + +See [CONTRIBUTING.md](../../CONTRIBUTING.md) for contribution guidelines. All commits must follow [Conventional Commits](https://www.conventionalcommits.org/). + +## Related Documentation + +- [Docling Serve Documentation](https://github.com/docling-project/docling-serve) +- [Quarkus Documentation](https://quarkus.io/guides/) +- [Docling Java API Documentation](../../docs/) diff --git a/docling-testing/docling-version-tests/build.gradle.kts b/docling-testing/docling-version-tests/build.gradle.kts index 4b3960c4..5cec8297 100644 --- a/docling-testing/docling-version-tests/build.gradle.kts +++ b/docling-testing/docling-version-tests/build.gradle.kts @@ -20,8 +20,8 @@ dependencies { compileOnly(libs.quarkus.wiremock) - testImplementation("io.quarkus:quarkus-junit5") - testImplementation("io.quarkus:quarkus-junit5-mockito") + testImplementation("io.quarkus:quarkus-junit") + testImplementation("io.quarkus:quarkus-junit-mockito") testImplementation(libs.quarkus.wiremock.test) testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310") } diff --git a/docling-testing/docling-version-tests/images/architecture.puml b/docling-testing/docling-version-tests/images/architecture.puml new file mode 100644 index 00000000..833f0fdb --- /dev/null +++ b/docling-testing/docling-version-tests/images/architecture.puml @@ -0,0 +1,116 @@ +@startuml +!theme plain +skinparam defaultFontName Helvetica +skinparam shadowing false + +actor User +participant "GitHub Actions\n(.github/workflows/version-tests.yml)" as GHA +participant "VersionTestsCommand\n(Picocli CLI)" as CLI +participant "RegistryClientFactory" as Factory +participant "GHCRClient\n(REST Client)" as GHCR +participant "TagsTester" as Tester +participant "DoclingServeContainer\n(Testcontainers)" as Container +participant "DoclingServe\n(Docker)" as Serve +participant "ResultsHandlers" as Handlers +participant "MarkdownFileResultsHandler\n(Qute)" as Markdown +participant "GithubIssueResultsHandler\n(GitHub API)" as GitHub + +== Workflow Trigger == + +alt Manual (workflow_dispatch) + User -> GHA: Trigger workflow\n(with inputs) +else Scheduled (cron) + GHA -> GHA: Monday 3 AM UTC\n(scheduled trigger) +end + +== Build & Setup == + +GHA -> GHA: Setup Java ${{ env.JAVA_VERSION }} +GHA -> GHA: Setup Gradle +GHA -> GHA: Build :docling-version-tests +GHA -> GHA: Process tags and flags\n(parse inputs, set env vars) + +== Run Version Tests == + +GHA -> CLI: Execute quarkus-run.jar\n(with CLI args) + +note right of GHA + java -jar build/quarkus-app/quarkus-run.jar + -p ${PARALLELISM} + -i ${IMAGE} + -r ${REGISTRY} + -o ${OUTPUT_DIR} + -e ${EXCLUDE_TAGS_REGEX} + --cleanup-container-images + ${TAGS} ${CREATE_GITHUB_ISSUE_FLAG} +end note + +== 1. Fetch Tags (if not provided) == + +CLI -> Factory: Get registry client +Factory -> GHCR: Create client +CLI -> GHCR: Request authentication token +GHCR -> GHCR: POST /token?scope=repository:{image}:pull +GHCR --> CLI: Return token + +CLI -> GHCR: Fetch available tags +GHCR -> GHCR: GET /v2/{image}/tags/list +GHCR --> CLI: Return tag list +CLI -> CLI: Filter semantic versions\nApply exclusion regex + +== 2. Test Each Tag in Parallel == + +loop For each tag + CLI -> Tester: Test tag version + Tester -> Container: Create container config\n(image, tag, timeout) + Tester -> Container: Start container + Container -> Serve: Pull and start\nDocker container + Serve --> Container: Container running + + Tester -> Serve: Health check\nGET /health + Serve --> Tester: {"status": "ok"} + + Tester -> Serve: Convert test document\n(source: https://docling.ai) + Serve --> Tester: Conversion response + + Tester -> Tester: Validate response:\n• No errors\n• Formats present\n• Deserialize JSON + + Tester -> Container: Capture logs + Container --> Tester: Container logs + + Tester -> Container: Stop and remove\n(cleanup enabled) + Container -> Serve: Stop container + + Tester --> CLI: Test result +end + +== 3. Handle Results == + +CLI -> Handlers: Process results + +Handlers -> Markdown: Generate report +Markdown -> Markdown: Render Qute template +Markdown --> Handlers: Write results.md + +alt test failures && GitHub issue enabled + Handlers -> GitHub: Create issue + GitHub -> GitHub: POST /repos/{owner}/{repo}/issues + GitHub --> Handlers: Issue created +end + +Handlers --> CLI: Results handled +CLI --> GHA: Exit with status + +== Publish Results == + +GHA -> GHA: Copy results.md\nto serve-compatibility.md + +GHA -> GHA: Create Pull Request\n(update compatibility table) + +alt PR created + GHA -> GHA: Auto-merge PR\n(--merge --admin) +end + +GHA --> User: Workflow complete + +@enduml \ No newline at end of file diff --git a/docling-testing/docling-version-tests/src/main/java/ai/docling/client/tester/service/TagsTester.java b/docling-testing/docling-version-tests/src/main/java/ai/docling/client/tester/service/TagsTester.java index 7def1ca1..2720975c 100644 --- a/docling-testing/docling-version-tests/src/main/java/ai/docling/client/tester/service/TagsTester.java +++ b/docling-testing/docling-version-tests/src/main/java/ai/docling/client/tester/service/TagsTester.java @@ -12,8 +12,6 @@ import org.assertj.core.api.InstanceOfAssertFactories; import org.testcontainers.DockerClientFactory; -import io.quarkus.logging.Log; - import ai.docling.client.tester.domain.TagTestResult; import ai.docling.client.tester.domain.TagTestResult.Result; import ai.docling.client.tester.domain.TagsTestRequest; @@ -27,10 +25,11 @@ import ai.docling.serve.api.convert.response.ConvertDocumentResponse; import ai.docling.serve.api.convert.response.DocumentResponse; import ai.docling.serve.api.health.HealthCheckResponse; -import ai.docling.serve.client.DoclingServeClientBuilderFactory; import ai.docling.testcontainers.serve.DoclingServeContainer; import ai.docling.testcontainers.serve.config.DoclingServeContainerConfig; +import io.quarkus.logging.Log; + @ApplicationScoped public class TagsTester { public TagsTestResults testTags(TagsTestRequest request) { @@ -90,7 +89,7 @@ private TagTestResult testTag(TagsTestRequest request, String tag) { } private void doConversion(DoclingServeContainer doclingContainer) { - var doclingClient = DoclingServeClientBuilderFactory.newBuilder() + var doclingClient = DoclingServeApi.builder() .baseUrl("http://localhost:%d".formatted(doclingContainer.getPort())) .build();