diff --git a/.gitignore b/.gitignore index 7b70f42..835ee8f 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ dist/ venv/ .tox/ .docker/ +docs_sphinx/venv/ +docs_sphinx/build/ diff --git a/docs/JSON.md b/docs/JSON.md new file mode 100644 index 0000000..8338249 --- /dev/null +++ b/docs/JSON.md @@ -0,0 +1,125 @@ + +# JSON Data Source + +> **Tip** +> See [examples/](../examples/) for sample configuration files. + +## Overview + +`JsonImporter` reads benchmark results from a local JSON file and feeds them into Otava for change-point analysis. It is a simple data source to set up — no external database or service is required. + +The importer caches parsed file content in memory, so a file is only read once per session even if multiple tests reference the same path. + +--- + +## Expected JSON Format + +The input file must be a JSON array. Each element represents a single benchmark run. +```json +[ + { + "timestamp": 1711929600, + "metrics": [ + { "name": "throughput", "value": 4821.0 }, + { "name": "p99_latency_ms", "value": 142.7 } + ], + "attributes": { + "branch": "main", + "commit": "a3f9c12" + } + }, + { + "timestamp": 1712016000, + "metrics": [ + { "name": "throughput", "value": 5013.0 }, + { "name": "p99_latency_ms", "value": 138.2 } + ], + "attributes": { + "branch": "main", + "commit": "b7d2e45" + } + } +] +``` + +--- + +## Fields + +### `timestamp` + +- **Type:** integer (Unix epoch seconds) +- **Required:** yes +- Identifies when the commit was merged into the tracked branch. This timestamp should remain constant for the same commit, even if benchmarks are rerun multiple times. + +### `metrics` + +- **Type:** array of objects +- **Required:** yes +- Each object must have: + - `name` (string) — unique identifier for the metric within this result + - `value` (number) — the measured value +- Metric names must be consistent across results for change-point analysis to be meaningful. + +> Note: A `unit` field (e.g., "ms") is not currently supported by JsonImporter. + +### `attributes` + +- **Type:** object (string → string) +- **Required:** no +- Arbitrary key-value pairs describing the run context (e.g. branch, commit, version). +- The `branch` key is required only when using branch-based filtering. +--- + +## Configuration Example + +Add a test with `type: json` to your `otava.yaml`: +```yaml +tests: + my_benchmark: + type: json + file: otava/examples/json/data/sample.json + base_branch: main +``` + +| Field | Required | Description | +|---|---|---| +| `type` | yes | Must be `json` | +| `file` | yes | Path to the JSON file | +| `base_branch` | no | If set, only runs from this branch are analyzed by default | + +--- + +## Limitations + +- The entire file is read into memory at once. Very large files may cause high memory usage. +- There is no schema validation. Missing or malformed fields will cause a `KeyError` at runtime. +- The `branch` filter requires the key `"branch"` to exist inside `attributes` on every entry — if it is absent on any entry that would otherwise be included, the importer will raise a `KeyError`. +- Attribute values are expected to be strings. No type coercion is performed. +- The file path is resolved at config load time; a missing file raises a `TestConfigError` immediately. + +--- + +## Example Usage + +Analyze test results stored in JSON format: +```bash +otava analyze my_benchmark --config otava/examples/json/config/otava.yaml +``` diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..7c8201e --- /dev/null +++ b/docs/index.html @@ -0,0 +1,100 @@ + + + + + + Apache Otava Documentation + + + + + +
+ +

Apache Otava Documentation

+ +

+ Structured entry point to the Otava documentation, replacing the default directory listing. +

+ +
+

Documentation

+ +
+ +
+

Project

+ +
+ +
+ + + \ No newline at end of file diff --git a/docs_sphinx/Makefile b/docs_sphinx/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs_sphinx/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs_sphinx/make.bat b/docs_sphinx/make.bat new file mode 100644 index 0000000..747ffb7 --- /dev/null +++ b/docs_sphinx/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs_sphinx/source/conf.py b/docs_sphinx/source/conf.py new file mode 100644 index 0000000..4d9cc2a --- /dev/null +++ b/docs_sphinx/source/conf.py @@ -0,0 +1,36 @@ +# Configuration file for the Sphinx documentation builder. +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) + +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'otava' +copyright = '2026, Apache Otava' +author = 'Apache Otava' +release = '0.1' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ["myst_parser"] + +templates_path = ['_templates'] +exclude_patterns = [] + + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'alabaster' +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", +} +html_static_path = ['_static'] diff --git a/docs_sphinx/source/index.rst b/docs_sphinx/source/index.rst new file mode 100644 index 0000000..b49957b --- /dev/null +++ b/docs_sphinx/source/index.rst @@ -0,0 +1,18 @@ +Welcome to Apache Otava Documentation +==================================== + +.. toctree:: + :maxdepth: 2 + + ../../docs/README.md + ../../docs/INSTALL.md + ../../docs/GETTING_STARTED.md + ../../docs/BASICS.md + ../../docs/CSV.md + ../../docs/BIG_QUERY.md + ../../docs/POSTGRESQL.md + ../../docs/GRAFANA.md + ../../docs/GRAPHITE.md + + +This documentation is powered by Sphinx and supports Markdown via MyST. diff --git a/examples/json/data/sample.json b/examples/json/data/sample.json new file mode 100644 index 0000000..3ed405c --- /dev/null +++ b/examples/json/data/sample.json @@ -0,0 +1,82 @@ +[ + { + "timestamp": 1767222000, + "metrics": [ + { "name": "metric1", "value": 154023 }, + { "name": "metric2", "value": 10.43 } + ], + "attributes": { "branch": "main", "commit": "aaa0" } + }, + { + "timestamp": 1767308400, + "metrics": [ + { "name": "metric1", "value": 138455 }, + { "name": "metric2", "value": 10.23 } + ], + "attributes": { "branch": "main", "commit": "aaa1" } + }, + { + "timestamp": 1767394800, + "metrics": [ + { "name": "metric1", "value": 143112 }, + { "name": "metric2", "value": 10.29 } + ], + "attributes": { "branch": "main", "commit": "aaa2" } + }, + { + "timestamp": 1767481200, + "metrics": [ + { "name": "metric1", "value": 149190 }, + { "name": "metric2", "value": 10.91 } + ], + "attributes": { "branch": "main", "commit": "aaa3" } + }, + { + "timestamp": 1767567600, + "metrics": [ + { "name": "metric1", "value": 132098 }, + { "name": "metric2", "value": 10.34 } + ], + "attributes": { "branch": "main", "commit": "aaa4" } + }, + { + "timestamp": 1767654000, + "metrics": [ + { "name": "metric1", "value": 151344 }, + { "name": "metric2", "value": 10.69 } + ], + "attributes": { "branch": "main", "commit": "aaa5" } + }, + { + "timestamp": 1767740400, + "metrics": [ + { "name": "metric1", "value": 155145 }, + { "name": "metric2", "value": 9.23 } + ], + "attributes": { "branch": "main", "commit": "aaa6" } + }, + { + "timestamp": 1767826800, + "metrics": [ + { "name": "metric1", "value": 148889 }, + { "name": "metric2", "value": 9.11 } + ], + "attributes": { "branch": "main", "commit": "aaa7" } + }, + { + "timestamp": 1767913200, + "metrics": [ + { "name": "metric1", "value": 149466 }, + { "name": "metric2", "value": 9.13 } + ], + "attributes": { "branch": "main", "commit": "aaa8" } + }, + { + "timestamp": 1767999600, + "metrics": [ + { "name": "metric1", "value": 148209 }, + { "name": "metric2", "value": 9.03 } + ], + "attributes": { "branch": "main", "commit": "aaa9" } + } +] diff --git a/otava/examples/json/config/otava.yaml b/otava/examples/json/config/otava.yaml new file mode 100644 index 0000000..9ec882d --- /dev/null +++ b/otava/examples/json/config/otava.yaml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +tests: + my_benchmark: + type: json + file: otava/examples/json/data/sample.json + base_branch: main diff --git a/otava/examples/json/data/sample.json b/otava/examples/json/data/sample.json new file mode 100644 index 0000000..f6816e6 --- /dev/null +++ b/otava/examples/json/data/sample.json @@ -0,0 +1,20 @@ +[ + { + "timestamp": 1711929600, + "metrics": [ + { "name": "throughput", "value": 100 } + ], + "attributes": { + "branch": "main" + } + }, + { + "timestamp": 1712016000, + "metrics": [ + { "name": "throughput", "value": 120 } + ], + "attributes": { + "branch": "main" + } + } +]